Retry rolling upgrade junit tests (#99760)

Re-applies the changes from #99572 to move some bwc tests to a junit-based build infrastructure. Some tests that did not handle the move well have been kept in rolling-upgrade-legacy using the old gradle-based infrastructure

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java

@@ -66,7 +66,7 @@ public abstract class RestrictedBuildApiService implements BuildService<Restrict
         map.put(LegacyRestTestBasePlugin.class, ":qa:multi-cluster-search");
         map.put(LegacyRestTestBasePlugin.class, ":qa:remote-clusters");
         map.put(LegacyRestTestBasePlugin.class, ":qa:repository-multi-version");
-        map.put(LegacyRestTestBasePlugin.class, ":qa:rolling-upgrade");
+        map.put(LegacyRestTestBasePlugin.class, ":qa:rolling-upgrade-legacy");
         map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-http");
         map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-ingest-disabled");
         map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-ingest-with-all-dependencies");

build-tools-internal/src/main/resources/checkstyle_suppressions.xml

@@ -32,7 +32,7 @@
 
   <!-- Intentionally have multi line string for a bulk request, otherwise this needs to fallback to string concatenation  -->
   <suppress files="modules[/\\]data-streams[/\\]src[/\\]javaRestTest[/\\]java[/\\]org[/\\]elasticsearch[/\\]datastreams[/\\]TsdbDataStreamRestIT.java" checks="LineLength" />
-  <suppress files="qa[/\\]rolling-upgrade[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]upgrades[/\\]TsdbIT.java" checks="LineLength" />
+  <suppress files="qa[/\\]rolling-upgrade[/\\]src[/\\]javaRestTest[/\\]java[/\\]org[/\\]elasticsearch[/\\]upgrades[/\\]TsdbIT.java" checks="LineLength" />
 
   <!-- Gradle requires inputs to be seriablizable -->
   <suppress files="build-tools-internal[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gradle[/\\]internal[/\\]precommit[/\\]TestingConventionRule.java" checks="RegexpSinglelineJava" />

qa/full-cluster-restart/build.gradle

@@ -19,3 +19,8 @@ BuildParams.bwcVersions.withIndexCompatible { bwcVersion, baseName ->
     systemProperty("tests.old_cluster_version", bwcVersion)
   }
 }
+
+tasks.withType(Test).configureEach {
+  // CI doesn't like it when there's multiple clusters running at once
+  maxParallelForks = 1
+}

qa/rolling-upgrade-legacy/build.gradle

@@ -0,0 +1,143 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+
+import org.elasticsearch.gradle.Version
+import org.elasticsearch.gradle.internal.BwcVersions
+import org.elasticsearch.gradle.internal.info.BuildParams
+import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
+
+apply plugin: 'elasticsearch.internal-testclusters'
+apply plugin: 'elasticsearch.standalone-rest-test'
+apply plugin: 'elasticsearch.bwc-test'
+apply plugin: 'elasticsearch.rest-resources'
+
+BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName ->
+  /*
+   * NOTE: This module is for the tests that were problematic when converting :qa:rolling-upgrade to the junit-based bwc test definition
+   * Over time, these should be migrated into the :qa:rolling-upgrade module and fixed properly
+   *
+   * The goal here is to:
+   * <ul>
+   *  <li>start three nodes on the old version
+   *  <li>run tests with systemProperty 'tests.rest.suite', 'old_cluster'
+   *  <li>upgrade one node
+   *  <li>run tests with systemProperty 'tests.rest.suite', 'mixed_cluster'
+   *  <li>upgrade one more node
+   *  <li>run tests with systemProperty 'tests.rest.suite', 'mixed_cluster' again
+   *  <li>updgrade the last node
+   *  <li>run tests with systemProperty 'tests.rest.suite', 'upgraded_cluster'
+   * </ul>
+   */
+
+  def baseCluster = testClusters.register(baseName) {
+    versions = [bwcVersion.toString(), project.version]
+    numberOfNodes = 3
+
+    setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
+    setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
+    setting 'xpack.security.enabled', 'false'
+    setting 'logger.org.elasticsearch.cluster.service.MasterService', 'TRACE'
+    setting 'logger.org.elasticsearch.cluster.routing.allocation.allocator.DesiredBalanceShardsAllocator', 'TRACE'
+    setting 'logger.org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders', 'TRACE'
+    requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0")
+  }
+
+  String oldVersion = bwcVersion.toString()
+
+  tasks.register("${baseName}#oldClusterTest", StandaloneRestIntegTestTask) {
+    dependsOn "processTestResources"
+    useCluster baseCluster
+    mustRunAfter("precommit")
+    doFirst {
+      delete("${buildDir}/cluster/shared/repo/${baseName}")
+    }
+    def excludeList = []
+    systemProperty 'tests.rest.suite', 'old_cluster'
+    systemProperty 'tests.upgrade_from_version', oldVersion
+    nonInputProperties.systemProperty('tests.rest.cluster', baseCluster.map(c -> c.allHttpSocketURI.join(",")))
+    nonInputProperties.systemProperty('tests.clustername', baseName)
+    if (bwcVersion.before("8.4.0")) {
+      excludeList.addAll(["old_cluster/30_vector_search/*"])
+    } else if (bwcVersion.before("8.6.0")) {
+      excludeList.addAll(["old_cluster/30_vector_search/Create indexed byte vectors and search"])
+    }
+    if (excludeList.isEmpty() == false) {
+      systemProperty 'tests.rest.blacklist', excludeList.join(',')
+    }
+  }
+
+  tasks.register("${baseName}#oneThirdUpgradedTest", StandaloneRestIntegTestTask) {
+    dependsOn "${baseName}#oldClusterTest"
+    useCluster baseCluster
+    doFirst {
+      baseCluster.get().nextNodeToNextVersion()
+    }
+    systemProperty 'tests.rest.suite', 'mixed_cluster'
+    systemProperty 'tests.upgrade_from_version', oldVersion
+    systemProperty 'tests.first_round', 'true'
+    nonInputProperties.systemProperty('tests.rest.cluster', baseCluster.map(c -> c.allHttpSocketURI.join(",")))
+    nonInputProperties.systemProperty('tests.clustername', baseName)
+    def excludeList = []
+    if (bwcVersion.before("8.4.0")) {
+      excludeList.addAll(["mixed_cluster/30_vector_search/*"])
+    } else if (bwcVersion.before("8.6.0")) {
+      excludeList.addAll(["mixed_cluster/30_vector_search/Search byte indices created in old cluster"])
+    }
+    if (excludeList.isEmpty() == false) {
+      systemProperty 'tests.rest.blacklist', excludeList.join(',')
+    }
+  }
+
+  tasks.register("${baseName}#twoThirdsUpgradedTest", StandaloneRestIntegTestTask) {
+    dependsOn "${baseName}#oneThirdUpgradedTest"
+    useCluster baseCluster
+    doFirst {
+      baseCluster.get().nextNodeToNextVersion()
+    }
+    systemProperty 'tests.rest.suite', 'mixed_cluster'
+    systemProperty 'tests.upgrade_from_version', oldVersion
+    systemProperty 'tests.first_round', 'false'
+    nonInputProperties.systemProperty('tests.rest.cluster', baseCluster.map(c -> c.allHttpSocketURI.join(",")))
+    nonInputProperties.systemProperty('tests.clustername', baseName)
+    def excludeList = []
+    if (bwcVersion.before("8.4.0")) {
+      excludeList.addAll(["mixed_cluster/30_vector_search/*"])
+    } else if (bwcVersion.before("8.6.0")) {
+      excludeList.addAll(["mixed_cluster/30_vector_search/Search byte indices created in old cluster"])
+    }
+    if (excludeList.isEmpty() == false) {
+      systemProperty 'tests.rest.blacklist', excludeList.join(',')
+    }
+  }
+
+  tasks.register("${baseName}#upgradedClusterTest", StandaloneRestIntegTestTask) {
+    dependsOn "${baseName}#twoThirdsUpgradedTest"
+    doFirst {
+      baseCluster.get().nextNodeToNextVersion()
+    }
+    useCluster testClusters.named(baseName)
+    systemProperty 'tests.rest.suite', 'upgraded_cluster'
+    systemProperty 'tests.upgrade_from_version', oldVersion
+    nonInputProperties.systemProperty('tests.rest.cluster', baseCluster.map(c -> c.allHttpSocketURI.join(",")))
+    nonInputProperties.systemProperty('tests.clustername', baseName)
+    def excludeList = []
+    if (bwcVersion.before("8.4.0")) {
+      excludeList.addAll(["upgraded_cluster/30_vector_search/*"])
+    } else if (bwcVersion.before("8.6.0")) {
+      excludeList.addAll(["upgraded_cluster/30_vector_search/Search byte indices created in old cluster"])
+    }
+    if (excludeList.isEmpty() == false) {
+      systemProperty 'tests.rest.blacklist', excludeList.join(',')
+    }
+  }
+
+  tasks.register(bwcTaskName(bwcVersion)) {
+    dependsOn tasks.named("${baseName}#upgradedClusterTest")
+  }
+}

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractRollingTestCase.java → qa/rolling-upgrade-legacy/src/test/java/org/elasticsearch/upgrades/AbstractRollingTestCase.java

@@ -9,11 +9,8 @@ package org.elasticsearch.upgrades;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.test.rest.ESRestTestCase;
 
-import static org.hamcrest.Matchers.lessThan;
-
 public abstract class AbstractRollingTestCase extends ESRestTestCase {
     protected enum ClusterType {
         OLD,
@@ -34,16 +31,6 @@ public abstract class AbstractRollingTestCase extends ESRestTestCase {
     protected static final boolean FIRST_MIXED_ROUND = Boolean.parseBoolean(System.getProperty("tests.first_round", "false"));
     protected static final Version UPGRADE_FROM_VERSION = Version.fromString(System.getProperty("tests.upgrade_from_version"));
 
-    protected static IndexVersion getOldClusterIndexVersion() {
-        var version = UPGRADE_FROM_VERSION;
-        if (version.equals(org.elasticsearch.Version.CURRENT)) {
-            return IndexVersion.current();
-        } else {
-            assertThat("Index version needs to be added to rolling test parameters", version, lessThan(org.elasticsearch.Version.V_8_11_0));
-            return IndexVersion.fromId(version.id);
-        }
-    }
-
     @Override
     protected final boolean resetFeatureStates() {
         return false;
@@ -54,11 +41,6 @@ public abstract class AbstractRollingTestCase extends ESRestTestCase {
         return true;
     }
 
-    @Override
-    protected final boolean preserveDataStreamsUponCompletion() {
-        return true;
-    }
-
     @Override
     protected final boolean preserveReposUponCompletion() {
         return true;

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java → qa/rolling-upgrade-legacy/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java

@@ -41,7 +41,6 @@ import static com.carrotsearch.randomizedtesting.RandomizedTest.randomAsciiLette
 import static org.elasticsearch.cluster.routing.UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING;
 import static org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider.INDEX_ROUTING_ALLOCATION_ENABLE_SETTING;
 import static org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY;
-import static org.elasticsearch.upgrades.UpgradeWithOldIndexSettingsIT.updateIndexSettingsPermittingSlowlogDeprecationWarning;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.in;
@@ -747,4 +746,23 @@ public class RecoveryIT extends AbstractRollingTestCase {
         ensureGreen(indexName);
         indexDocs(indexName, randomInt(100), randomInt(100));
     }
+
+    /*
+     * Copied from UpgradeWithOldIndexSettingsIT in the new format
+     */
+    private static void updateIndexSettingsPermittingSlowlogDeprecationWarning(String index, Settings.Builder settings) throws IOException {
+        Request request = new Request("PUT", "/" + index + "/_settings");
+        request.setJsonEntity(org.elasticsearch.common.Strings.toString(settings.build()));
+        if (UPGRADE_FROM_VERSION.before(Version.V_7_17_9)) {
+            // There is a bug (fixed in 7.17.9 and 8.7.0 where deprecation warnings could leak into ClusterApplierService#applyChanges)
+            // Below warnings are set (and leaking) from an index in this test case
+            request.setOptions(expectVersionSpecificWarnings(v -> {
+                v.compatible(
+                    "[index.indexing.slowlog.level] setting was deprecated in Elasticsearch and will be removed in a future release! "
+                        + "See the breaking changes documentation for the next major version."
+                );
+            }));
+        }
+        client().performRequest(request);
+    }
 }

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java → qa/rolling-upgrade-legacy/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java

@@ -40,11 +40,6 @@ public class UpgradeClusterClientYamlTestSuiteIT extends ESClientYamlSuiteTestCa
         return true;
     }
 
-    @Override
-    protected boolean preserveDataStreamsUponCompletion() {
-        return true;
-    }
-
     public UpgradeClusterClientYamlTestSuiteIT(ClientYamlTestCandidate testCandidate) {
         super(testCandidate);
     }

qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_vector_search.yml → qa/rolling-upgrade-legacy/src/test/resources/rest-api-spec/test/old_cluster/30_vector_search.yml

@@ -11,7 +11,6 @@
               bdv:
                 type: dense_vector
                 dims: 3
-                index: false
               knn:
                 type: dense_vector
                 dims: 3
@@ -126,7 +125,6 @@
               bdv:
                 type: dense_vector
                 element_type: byte
-                index: false
                 dims: 3
               knn:
                 type: dense_vector

qa/rolling-upgrade/build.gradle

@@ -6,135 +6,26 @@
  * Side Public License, v 1.
  */
 
-
-import org.elasticsearch.gradle.Version
-import org.elasticsearch.gradle.internal.BwcVersions
 import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
 
 apply plugin: 'elasticsearch.internal-testclusters'
-apply plugin: 'elasticsearch.standalone-rest-test'
+apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.internal-test-artifact-base'
 apply plugin: 'elasticsearch.bwc-test'
-apply plugin: 'elasticsearch.rest-resources'
-
-BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName ->
-  /*
-   * The goal here is to:
-   * <ul>
-   *  <li>start three nodes on the old version
-   *  <li>run tests with systemProperty 'tests.rest.suite', 'old_cluster'
-   *  <li>upgrade one node
-   *  <li>run tests with systemProperty 'tests.rest.suite', 'mixed_cluster'
-   *  <li>upgrade one more node
-   *  <li>run tests with systemProperty 'tests.rest.suite', 'mixed_cluster' again
-   *  <li>updgrade the last node
-   *  <li>run tests with systemProperty 'tests.rest.suite', 'upgraded_cluster'
-   * </ul>
-   */
-
-  def baseCluster = testClusters.register(baseName) {
-    versions = [bwcVersion.toString(), project.version]
-    numberOfNodes = 3
-
-    setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
-    setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
-    setting 'xpack.security.enabled', 'false'
-    setting 'logger.org.elasticsearch.cluster.service.MasterService', 'TRACE'
-    setting 'logger.org.elasticsearch.cluster.routing.allocation.allocator.DesiredBalanceShardsAllocator', 'TRACE'
-    setting 'logger.org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders', 'TRACE'
-    requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0")
-  }
-
-  String oldVersion = bwcVersion.toString()
-
-  tasks.register("${baseName}#oldClusterTest", StandaloneRestIntegTestTask) {
-    dependsOn "processTestResources"
-    useCluster baseCluster
-    mustRunAfter("precommit")
-    doFirst {
-      delete("${buildDir}/cluster/shared/repo/${baseName}")
-    }
-    def excludeList = []
-    systemProperty 'tests.rest.suite', 'old_cluster'
-    systemProperty 'tests.upgrade_from_version', oldVersion
-    nonInputProperties.systemProperty('tests.rest.cluster', baseCluster.map(c -> c.allHttpSocketURI.join(",")))
-    nonInputProperties.systemProperty('tests.clustername', baseName)
-    if (bwcVersion.before("8.4.0")) {
-      excludeList.addAll(["old_cluster/30_vector_search/*"])
-    } else if (bwcVersion.before("8.6.0")) {
-      excludeList.addAll(["old_cluster/30_vector_search/Create indexed byte vectors and search"])
-    }
-    if (excludeList.isEmpty() == false) {
-      systemProperty 'tests.rest.blacklist', excludeList.join(',')
-    }
-  }
-
-  tasks.register("${baseName}#oneThirdUpgradedTest", StandaloneRestIntegTestTask) {
-    dependsOn "${baseName}#oldClusterTest"
-    useCluster baseCluster
-    doFirst {
-      baseCluster.get().nextNodeToNextVersion()
-    }
-    systemProperty 'tests.rest.suite', 'mixed_cluster'
-    systemProperty 'tests.upgrade_from_version', oldVersion
-    systemProperty 'tests.first_round', 'true'
-    nonInputProperties.systemProperty('tests.rest.cluster', baseCluster.map(c -> c.allHttpSocketURI.join(",")))
-    nonInputProperties.systemProperty('tests.clustername', baseName)
-    def excludeList = []
-    if (bwcVersion.before("8.4.0")) {
-      excludeList.addAll(["mixed_cluster/30_vector_search/*"])
-    } else if (bwcVersion.before("8.6.0")) {
-      excludeList.addAll(["mixed_cluster/30_vector_search/Search byte indices created in old cluster"])
-    }
-    if (excludeList.isEmpty() == false) {
-      systemProperty 'tests.rest.blacklist', excludeList.join(',')
-    }
-  }
 
-  tasks.register("${baseName}#twoThirdsUpgradedTest", StandaloneRestIntegTestTask) {
-    dependsOn "${baseName}#oneThirdUpgradedTest"
-    useCluster baseCluster
-    doFirst {
-      baseCluster.get().nextNodeToNextVersion()
-    }
-    systemProperty 'tests.rest.suite', 'mixed_cluster'
-    systemProperty 'tests.upgrade_from_version', oldVersion
-    systemProperty 'tests.first_round', 'false'
-    nonInputProperties.systemProperty('tests.rest.cluster', baseCluster.map(c -> c.allHttpSocketURI.join(",")))
-    nonInputProperties.systemProperty('tests.clustername', baseName)
-    def excludeList = []
-    if (bwcVersion.before("8.4.0")) {
-      excludeList.addAll(["mixed_cluster/30_vector_search/*"])
-    } else if (bwcVersion.before("8.6.0")) {
-      excludeList.addAll(["mixed_cluster/30_vector_search/Search byte indices created in old cluster"])
-    }
-    if (excludeList.isEmpty() == false) {
-      systemProperty 'tests.rest.blacklist', excludeList.join(',')
-    }
-  }
+testArtifacts {
+  registerTestArtifactFromSourceSet(sourceSets.javaRestTest)
+}
 
-  tasks.register("${baseName}#upgradedClusterTest", StandaloneRestIntegTestTask) {
-    dependsOn "${baseName}#twoThirdsUpgradedTest"
-    doFirst {
-      baseCluster.get().nextNodeToNextVersion()
-    }
-    useCluster testClusters.named(baseName)
-    systemProperty 'tests.rest.suite', 'upgraded_cluster'
-    systemProperty 'tests.upgrade_from_version', oldVersion
-    nonInputProperties.systemProperty('tests.rest.cluster', baseCluster.map(c -> c.allHttpSocketURI.join(",")))
-    nonInputProperties.systemProperty('tests.clustername', baseName)
-    def excludeList = []
-    if (bwcVersion.before("8.4.0")) {
-      excludeList.addAll(["upgraded_cluster/30_vector_search/*"])
-    } else if (bwcVersion.before("8.6.0")) {
-      excludeList.addAll(["upgraded_cluster/30_vector_search/Search byte indices created in old cluster"])
-    }
-    if (excludeList.isEmpty() == false) {
-      systemProperty 'tests.rest.blacklist', excludeList.join(',')
-    }
+BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName ->
+  tasks.register(bwcTaskName(bwcVersion), StandaloneRestIntegTestTask) {
+    usesBwcDistribution(bwcVersion)
+    systemProperty("tests.old_cluster_version", bwcVersion)
   }
+}
 
-  tasks.register(bwcTaskName(bwcVersion)) {
-    dependsOn tasks.named("${baseName}#upgradedClusterTest")
-  }
+tasks.withType(Test).configureEach {
+  // CI doesn't like it when there's multiple clusters running at once
+  maxParallelForks = 1
 }

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DesiredNodesUpgradeIT.java → qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DesiredNodesUpgradeIT.java

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.upgrades;
 
+import com.carrotsearch.randomizedtesting.annotations.Name;
+
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.desirednodes.UpdateDesiredNodesRequest;
 import org.elasticsearch.client.Request;
@@ -31,18 +33,26 @@ import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.is;
 
-public class DesiredNodesUpgradeIT extends AbstractRollingTestCase {
+public class DesiredNodesUpgradeIT extends ParameterizedRollingUpgradeTestCase {
+
+    private final int desiredNodesVersion;
+
+    public DesiredNodesUpgradeIT(@Name("upgradedNodes") int upgradedNodes) {
+        super(upgradedNodes);
+        desiredNodesVersion = upgradedNodes + 1;
+    }
+
     private enum ProcessorsPrecision {
         DOUBLE,
         FLOAT
     }
 
     public void testUpgradeDesiredNodes() throws Exception {
-        assumeTrue("Desired nodes was introduced in 8.1", UPGRADE_FROM_VERSION.onOrAfter(Version.V_8_1_0));
+        assumeTrue("Desired nodes was introduced in 8.1", getOldClusterVersion().onOrAfter(Version.V_8_1_0));
 
-        if (UPGRADE_FROM_VERSION.onOrAfter(Processors.DOUBLE_PROCESSORS_SUPPORT_VERSION)) {
+        if (getOldClusterVersion().onOrAfter(Processors.DOUBLE_PROCESSORS_SUPPORT_VERSION)) {
             assertUpgradedNodesCanReadDesiredNodes();
-        } else if (UPGRADE_FROM_VERSION.onOrAfter(DesiredNode.RANGE_FLOAT_PROCESSORS_SUPPORT_VERSION)) {
+        } else if (getOldClusterVersion().onOrAfter(DesiredNode.RANGE_FLOAT_PROCESSORS_SUPPORT_VERSION)) {
             assertDesiredNodesUpdatedWithRoundedUpFloatsAreIdempotent();
         } else {
             assertDesiredNodesWithFloatProcessorsAreRejectedInOlderVersions();
@@ -50,13 +60,7 @@ public class DesiredNodesUpgradeIT extends AbstractRollingTestCase {
     }
 
     private void assertUpgradedNodesCanReadDesiredNodes() throws Exception {
-        final int desiredNodesVersion = switch (CLUSTER_TYPE) {
-            case OLD -> 1;
-            case MIXED -> FIRST_MIXED_ROUND ? 2 : 3;
-            case UPGRADED -> 4;
-        };
-
-        if (CLUSTER_TYPE != ClusterType.OLD) {
+        if (isMixedCluster() || isUpgradedCluster()) {
             final Map<String, Object> desiredNodes = getLatestDesiredNodes();
             final String historyId = extractValue(desiredNodes, "history_id");
             final int version = extractValue(desiredNodes, "version");
@@ -83,13 +87,7 @@ public class DesiredNodesUpgradeIT extends AbstractRollingTestCase {
             )
             .toList();
 
-        final int desiredNodesVersion = switch (CLUSTER_TYPE) {
-            case OLD -> 1;
-            case MIXED -> FIRST_MIXED_ROUND ? 2 : 3;
-            case UPGRADED -> 4;
-        };
-
-        if (CLUSTER_TYPE != ClusterType.OLD) {
+        if (isMixedCluster() || isUpgradedCluster()) {
             updateDesiredNodes(desiredNodes, desiredNodesVersion - 1);
         }
         for (int i = 0; i < 2; i++) {
@@ -100,28 +98,25 @@ public class DesiredNodesUpgradeIT extends AbstractRollingTestCase {
         final int latestDesiredNodesVersion = extractValue(latestDesiredNodes, "version");
         assertThat(latestDesiredNodesVersion, is(equalTo(desiredNodesVersion)));
 
-        if (CLUSTER_TYPE == ClusterType.UPGRADED) {
+        if (isUpgradedCluster()) {
             assertAllDesiredNodesAreActualized();
         }
     }
 
     private void assertDesiredNodesWithFloatProcessorsAreRejectedInOlderVersions() throws Exception {
-        switch (CLUSTER_TYPE) {
-            case OLD -> addClusterNodesToDesiredNodesWithIntegerProcessors(1);
-            case MIXED -> {
-                int version = FIRST_MIXED_ROUND ? 2 : 3;
-                // Processor ranges or float processors are forbidden during upgrades: 8.2 -> 8.3 clusters
-                final var responseException = expectThrows(
-                    ResponseException.class,
-                    () -> addClusterNodesToDesiredNodesWithProcessorsOrProcessorRanges(version, ProcessorsPrecision.FLOAT)
-                );
-                final var statusCode = responseException.getResponse().getStatusLine().getStatusCode();
-                assertThat(statusCode, is(equalTo(400)));
-            }
-            case UPGRADED -> {
-                assertAllDesiredNodesAreActualized();
-                addClusterNodesToDesiredNodesWithProcessorsOrProcessorRanges(4, ProcessorsPrecision.FLOAT);
-            }
+        if (isOldCluster()) {
+            addClusterNodesToDesiredNodesWithIntegerProcessors(1);
+        } else if (isMixedCluster()) {
+            // Processor ranges or float processors are forbidden during upgrades: 8.2 -> 8.3 clusters
+            final var responseException = expectThrows(
+                ResponseException.class,
+                () -> addClusterNodesToDesiredNodesWithProcessorsOrProcessorRanges(desiredNodesVersion, ProcessorsPrecision.FLOAT)
+            );
+            final var statusCode = responseException.getResponse().getStatusLine().getStatusCode();
+            assertThat(statusCode, is(equalTo(400)));
+        } else {
+            assertAllDesiredNodesAreActualized();
+            addClusterNodesToDesiredNodesWithProcessorsOrProcessorRanges(4, ProcessorsPrecision.FLOAT);
         }
 
         getLatestDesiredNodes();

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/FeatureUpgradeIT.java → qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FeatureUpgradeIT.java

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.upgrades;
 
+import com.carrotsearch.randomizedtesting.annotations.Name;
+
 import org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.ResponseException;
@@ -21,14 +23,17 @@ import static org.hamcrest.Matchers.aMapWithSize;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 
-public class FeatureUpgradeIT extends AbstractRollingTestCase {
+public class FeatureUpgradeIT extends ParameterizedRollingUpgradeTestCase {
+
+    public FeatureUpgradeIT(@Name("upgradedNodes") int upgradedNodes) {
+        super(upgradedNodes);
+    }
 
-    @SuppressWarnings("unchecked")
     public void testGetFeatureUpgradeStatus() throws Exception {
 
         final String systemIndexWarning = "this request accesses system indices: [.tasks], but in a future major version, direct "
             + "access to system indices will be prevented by default";
-        if (CLUSTER_TYPE == ClusterType.OLD) {
+        if (isOldCluster()) {
             // setup - put something in the tasks index
             // create index
             Request createTestIndex = new Request("PUT", "/feature_test_index_old");
@@ -79,7 +84,7 @@ public class FeatureUpgradeIT extends AbstractRollingTestCase {
                 }
             });
 
-        } else if (CLUSTER_TYPE == ClusterType.UPGRADED) {
+        } else if (isUpgradedCluster()) {
             // check results
             assertBusy(() -> {
                 Request clusterStateRequest = new Request("GET", "/_migration/system_features");
@@ -95,7 +100,7 @@ public class FeatureUpgradeIT extends AbstractRollingTestCase {
 
                 assertThat(feature, aMapWithSize(4));
                 assertThat(feature.get("minimum_index_version"), equalTo(getOldClusterIndexVersion().toString()));
-                if (UPGRADE_FROM_VERSION.before(TransportGetFeatureUpgradeStatusAction.NO_UPGRADE_REQUIRED_VERSION)) {
+                if (getOldClusterVersion().before(TransportGetFeatureUpgradeStatusAction.NO_UPGRADE_REQUIRED_VERSION)) {
                     assertThat(feature.get("migration_status"), equalTo("MIGRATION_NEEDED"));
                 } else {
                     assertThat(feature.get("migration_status"), equalTo("NO_MIGRATION_NEEDED"));
@@ -103,5 +108,4 @@ public class FeatureUpgradeIT extends AbstractRollingTestCase {
             });
         }
     }
-
 }

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/FieldCapsIT.java → qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FieldCapsIT.java

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.upgrades;
 
+import com.carrotsearch.randomizedtesting.annotations.Name;
+
 import org.apache.http.HttpHost;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
@@ -36,15 +38,17 @@ import static org.hamcrest.Matchers.equalTo;
  * In 8.2 we also added the ability to filter fields by type and metadata, with some post-hoc filtering applied on
  * the co-ordinating node if older nodes were included in the system
  */
-public class FieldCapsIT extends AbstractRollingTestCase {
-    private static boolean indicesCreated = false;
+public class FieldCapsIT extends ParameterizedRollingUpgradeTestCase {
+
+    public FieldCapsIT(@Name("upgradedNodes") int upgradedNodes) {
+        super(upgradedNodes);
+    }
+
+    private static boolean oldIndicesCreated;
+    private static boolean newIndicesCreated;
 
     @Before
     public void setupIndices() throws Exception {
-        if (indicesCreated) {
-            return;
-        }
-        indicesCreated = true;
         final String redMapping = """
              "properties": {
                "red_field": { "type": "keyword" },
@@ -63,7 +67,7 @@ public class FieldCapsIT extends AbstractRollingTestCase {
                "timestamp": {"type": "date"}
              }
             """;
-        if (CLUSTER_TYPE == ClusterType.OLD) {
+        if (isOldCluster() && oldIndicesCreated == false) {
             createIndex("old_red_1", Settings.EMPTY, redMapping);
             createIndex("old_red_2", Settings.EMPTY, redMapping);
             createIndex("old_red_empty", Settings.EMPTY, redMapping);
@@ -78,7 +82,8 @@ public class FieldCapsIT extends AbstractRollingTestCase {
                 );
                 assertOK(client().performRequest(indexRequest));
             }
-        } else if (CLUSTER_TYPE == ClusterType.MIXED && FIRST_MIXED_ROUND) {
+            oldIndicesCreated = true;
+        } else if (isFirstMixedCluster() && newIndicesCreated == false) {
             createIndex("new_red_1", Settings.EMPTY, redMapping);
             createIndex("new_red_2", Settings.EMPTY, redMapping);
             createIndex("new_red_empty", Settings.EMPTY, redMapping);
@@ -93,6 +98,7 @@ public class FieldCapsIT extends AbstractRollingTestCase {
                 );
                 assertOK(client().performRequest(indexRequest));
             }
+            newIndicesCreated = true;
         }
     }
 
@@ -149,7 +155,7 @@ public class FieldCapsIT extends AbstractRollingTestCase {
     }
 
     public void testNewIndicesOnly() throws Exception {
-        assumeFalse("required mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.OLD);
+        assumeFalse("required mixed or upgraded cluster", isOldCluster());
         {
             FieldCapabilitiesResponse resp = fieldCaps(List.of("new_red_*"), List.of("*"), null, null, null);
             assertThat(resp.getIndices(), equalTo(new String[] { "new_red_1", "new_red_2", "new_red_empty" }));
@@ -177,7 +183,7 @@ public class FieldCapsIT extends AbstractRollingTestCase {
     }
 
     public void testNewIndicesOnlyWithIndexFilter() throws Exception {
-        assumeFalse("required mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.OLD);
+        assumeFalse("required mixed or upgraded cluster", isOldCluster());
         final QueryBuilder indexFilter = QueryBuilders.rangeQuery("timestamp").gte("2020-01-01").lte("2020-12-12");
         {
             FieldCapabilitiesResponse resp = fieldCaps(List.of("new_red_*"), List.of("*"), indexFilter, null, null);
@@ -203,7 +209,7 @@ public class FieldCapsIT extends AbstractRollingTestCase {
     }
 
     public void testAllIndices() throws Exception {
-        assumeFalse("required mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.OLD);
+        assumeFalse("required mixed or upgraded cluster", isOldCluster());
         FieldCapabilitiesResponse resp = fieldCaps(List.of("old_*", "new_*"), List.of("*"), null, null, null);
         assertThat(
             resp.getIndices(),
@@ -235,7 +241,7 @@ public class FieldCapsIT extends AbstractRollingTestCase {
     }
 
     public void testAllIndicesWithIndexFilter() throws Exception {
-        assumeFalse("required mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.OLD);
+        assumeFalse("required mixed or upgraded cluster", isOldCluster());
         final QueryBuilder indexFilter = QueryBuilders.rangeQuery("timestamp").gte("2020-01-01").lte("2020-12-12");
         FieldCapabilitiesResponse resp = fieldCaps(List.of("old_*", "new_*"), List.of("*"), indexFilter, null, null);
         assertThat(
@@ -285,7 +291,7 @@ public class FieldCapsIT extends AbstractRollingTestCase {
     // because we are testing that the upgraded node will correctly apply filtering
     // to responses from older nodes that don't understand the filter parameters
     public void testAllIndicesWithFieldTypeFilter() throws Exception {
-        assumeFalse("required mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.OLD);
+        assumeFalse("required mixed or upgraded cluster", isOldCluster());
         RestClient restClient = getUpgradedNodeClient();
         FieldCapabilitiesResponse resp = fieldCaps(restClient, List.of("old_*", "new_*"), List.of("*"), null, "keyword", null);
         assertThat(resp.getField("red_field").keySet(), contains("keyword"));
@@ -298,7 +304,7 @@ public class FieldCapsIT extends AbstractRollingTestCase {
     // because we are testing that the upgraded node will correctly apply filtering
     // to responses from older nodes that don't understand the filter parameters
     public void testAllIndicesWithExclusionFilter() throws Exception {
-        assumeFalse("required mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.OLD);
+        assumeFalse("required mixed or upgraded cluster", isOldCluster());
         RestClient client = getUpgradedNodeClient();
         {
             FieldCapabilitiesResponse resp = fieldCaps(client, List.of("old_*", "new_*"), List.of("*"), null, null, null);

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java → qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/IndexingIT.java

@@ -7,6 +7,8 @@
  */
 package org.elasticsearch.upgrades;
 
+import com.carrotsearch.randomizedtesting.annotations.Name;
+
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.Version;
 import org.elasticsearch.client.Request;
@@ -15,7 +17,6 @@ import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.time.DateUtils;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
-import org.elasticsearch.core.Booleans;
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.test.ListMatcher;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -40,39 +41,36 @@ import static org.hamcrest.Matchers.equalTo;
 
 /**
  * Basic test that indexed documents survive the rolling restart. See
- * {@link RecoveryIT} for much more in depth testing of the mechanism
+ * {@code RecoveryIT} for much more in depth testing of the mechanism
  * by which they survive.
  * <p>
  * This test is an almost exact copy of <code>IndexingIT</code> in the
  * xpack rolling restart tests. We should work on a way to remove this
  * duplication but for now we have no real way to share code.
  */
-public class IndexingIT extends AbstractRollingTestCase {
+public class IndexingIT extends ParameterizedRollingUpgradeTestCase {
+
+    public IndexingIT(@Name("upgradedNodes") int upgradedNodes) {
+        super(upgradedNodes);
+    }
 
     public void testIndexing() throws IOException {
-        switch (CLUSTER_TYPE) {
-            case OLD:
-                break;
-            case MIXED:
-                Request waitForYellow = new Request("GET", "/_cluster/health");
-                waitForYellow.addParameter("wait_for_nodes", "3");
-                waitForYellow.addParameter("wait_for_status", "yellow");
-                client().performRequest(waitForYellow);
-                break;
-            case UPGRADED:
-                Request waitForGreen = new Request("GET", "/_cluster/health/test_index,index_with_replicas,empty_index");
-                waitForGreen.addParameter("wait_for_nodes", "3");
-                waitForGreen.addParameter("wait_for_status", "green");
-                // wait for long enough that we give delayed unassigned shards to stop being delayed
-                waitForGreen.addParameter("timeout", "70s");
-                waitForGreen.addParameter("level", "shards");
-                client().performRequest(waitForGreen);
-                break;
-            default:
-                throw new UnsupportedOperationException("Unknown cluster type [" + CLUSTER_TYPE + "]");
+        if (isMixedCluster()) {
+            Request waitForYellow = new Request("GET", "/_cluster/health");
+            waitForYellow.addParameter("wait_for_nodes", "3");
+            waitForYellow.addParameter("wait_for_status", "yellow");
+            client().performRequest(waitForYellow);
+        } else if (isUpgradedCluster()) {
+            Request waitForGreen = new Request("GET", "/_cluster/health/test_index,index_with_replicas,empty_index");
+            waitForGreen.addParameter("wait_for_nodes", "3");
+            waitForGreen.addParameter("wait_for_status", "green");
+            // wait for long enough that we give delayed unassigned shards to stop being delayed
+            waitForGreen.addParameter("timeout", "70s");
+            waitForGreen.addParameter("level", "shards");
+            client().performRequest(waitForGreen);
         }
 
-        if (CLUSTER_TYPE == ClusterType.OLD) {
+        if (isOldCluster()) {
             Request createTestIndex = new Request("PUT", "/test_index");
             createTestIndex.setJsonEntity("{\"settings\": {\"index.number_of_replicas\": 0}}");
             useIgnoreMultipleMatchingTemplatesWarningsHandler(createTestIndex);
@@ -95,30 +93,20 @@ public class IndexingIT extends AbstractRollingTestCase {
         }
 
         int expectedCount;
-        switch (CLUSTER_TYPE) {
-            case OLD:
-                expectedCount = 5;
-                break;
-            case MIXED:
-                if (Booleans.parseBoolean(System.getProperty("tests.first_round"))) {
-                    expectedCount = 5;
-                } else {
-                    expectedCount = 10;
-                }
-                break;
-            case UPGRADED:
-                expectedCount = 15;
-                break;
-            default:
-                throw new UnsupportedOperationException("Unknown cluster type [" + CLUSTER_TYPE + "]");
+        if (isOldCluster() || isFirstMixedCluster()) {
+            expectedCount = 5;
+        } else if (isMixedCluster()) {
+            expectedCount = 10;
+        } else {
+            expectedCount = 15;
         }
 
         assertCount("test_index", expectedCount);
         assertCount("index_with_replicas", 5);
         assertCount("empty_index", 0);
 
-        if (CLUSTER_TYPE != ClusterType.OLD) {
-            bulk("test_index", "_" + CLUSTER_TYPE, 5);
+        if (isOldCluster() == false) {
+            bulk("test_index", "_" + (isMixedCluster() ? "MIXED" : "UPGRADED"), 5);
             Request toBeDeleted = new Request("PUT", "/test_index/_doc/to_be_deleted");
             toBeDeleted.addParameter("refresh", "true");
             toBeDeleted.setJsonEntity("{\"f1\": \"delete-me\"}");
@@ -143,82 +131,76 @@ public class IndexingIT extends AbstractRollingTestCase {
         bulk.addParameter("refresh", "true");
         bulk.setJsonEntity(b);
 
-        switch (CLUSTER_TYPE) {
-            case OLD -> {
-                Request createTestIndex = new Request("PUT", "/" + indexName);
-                createTestIndex.setJsonEntity("{\"settings\": {\"index.number_of_replicas\": 0}}");
-                client().performRequest(createTestIndex);
-            }
-            case MIXED -> {
-                Request waitForGreen = new Request("GET", "/_cluster/health");
-                waitForGreen.addParameter("wait_for_nodes", "3");
-                client().performRequest(waitForGreen);
-                Version minNodeVersion = minNodeVersion();
-                if (minNodeVersion.before(Version.V_7_5_0)) {
-                    ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(bulk));
-                    assertEquals(400, e.getResponse().getStatusLine().getStatusCode());
-                    assertThat(
-                        e.getMessage(),
-                        // if request goes to 7.5+ node
-                        either(containsString("optype create not supported for indexing requests without explicit id until"))
-                            // if request goes to < 7.5 node
-                            .or(containsString("an id must be provided if version type or value are set"))
-                    );
-                } else {
-                    client().performRequest(bulk);
-                }
+        if (isOldCluster()) {
+            Request createTestIndex = new Request("PUT", "/" + indexName);
+            createTestIndex.setJsonEntity("{\"settings\": {\"index.number_of_replicas\": 0}}");
+            client().performRequest(createTestIndex);
+        } else if (isMixedCluster()) {
+            Request waitForGreen = new Request("GET", "/_cluster/health");
+            waitForGreen.addParameter("wait_for_nodes", "3");
+            client().performRequest(waitForGreen);
+            Version minNodeVersion = minNodeVersion();
+            if (minNodeVersion.before(Version.V_7_5_0)) {
+                ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(bulk));
+                assertEquals(400, e.getResponse().getStatusLine().getStatusCode());
+                assertThat(
+                    e.getMessage(),
+                    // if request goes to 7.5+ node
+                    either(containsString("optype create not supported for indexing requests without explicit id until"))
+                        // if request goes to < 7.5 node
+                        .or(containsString("an id must be provided if version type or value are set"))
+                );
+            } else {
+                client().performRequest(bulk);
             }
-            case UPGRADED -> client().performRequest(bulk);
-            default -> throw new UnsupportedOperationException("Unknown cluster type [" + CLUSTER_TYPE + "]");
+        } else if (isUpgradedCluster()) {
+            client().performRequest(bulk);
         }
     }
 
     public void testDateNanosFormatUpgrade() throws IOException {
         final String indexName = "test_date_nanos";
-        switch (CLUSTER_TYPE) {
-            case OLD -> {
-                Request createIndex = new Request("PUT", "/" + indexName);
-                XContentBuilder mappings = XContentBuilder.builder(XContentType.JSON.xContent())
-                    .startObject()
-                    .startObject("mappings")
-                    .startObject("properties")
-                    .startObject("date")
-                    .field("type", "date")
-                    .endObject()
-                    .startObject("date_nanos")
-                    .field("type", "date_nanos")
-                    .endObject()
-                    .endObject()
-                    .endObject()
-                    .endObject();
-                createIndex.setJsonEntity(Strings.toString(mappings));
-                client().performRequest(createIndex);
-                Request index = new Request("POST", "/" + indexName + "/_doc/");
-                XContentBuilder doc = XContentBuilder.builder(XContentType.JSON.xContent())
-                    .startObject()
-                    .field("date", "2015-01-01T12:10:30.123456789Z")
-                    .field("date_nanos", "2015-01-01T12:10:30.123456789Z")
-                    .endObject();
-                index.addParameter("refresh", "true");
-                index.setJsonEntity(Strings.toString(doc));
-                client().performRequest(index);
-            }
-            case UPGRADED -> {
-                Request search = new Request("POST", "/" + indexName + "/_search");
-                XContentBuilder query = XContentBuilder.builder(XContentType.JSON.xContent())
-                    .startObject()
-                    .array("fields", new String[] { "date", "date_nanos" })
-                    .endObject();
-                search.setJsonEntity(Strings.toString(query));
-                Map<String, Object> response = entityAsMap(client().performRequest(search));
-                Map<?, ?> bestHit = (Map<?, ?>) ((List<?>) (XContentMapValues.extractValue("hits.hits", response))).get(0);
-                List<?> date = (List<?>) XContentMapValues.extractValue("fields.date", bestHit);
-                assertThat(date.size(), equalTo(1));
-                assertThat(date.get(0), equalTo("2015-01-01T12:10:30.123Z"));
-                List<?> dateNanos = (List<?>) XContentMapValues.extractValue("fields.date_nanos", bestHit);
-                assertThat(dateNanos.size(), equalTo(1));
-                assertThat(dateNanos.get(0), equalTo("2015-01-01T12:10:30.123456789Z"));
-            }
+        if (isOldCluster()) {
+            Request createIndex = new Request("PUT", "/" + indexName);
+            XContentBuilder mappings = XContentBuilder.builder(XContentType.JSON.xContent())
+                .startObject()
+                .startObject("mappings")
+                .startObject("properties")
+                .startObject("date")
+                .field("type", "date")
+                .endObject()
+                .startObject("date_nanos")
+                .field("type", "date_nanos")
+                .endObject()
+                .endObject()
+                .endObject()
+                .endObject();
+            createIndex.setJsonEntity(Strings.toString(mappings));
+            client().performRequest(createIndex);
+            Request index = new Request("POST", "/" + indexName + "/_doc/");
+            XContentBuilder doc = XContentBuilder.builder(XContentType.JSON.xContent())
+                .startObject()
+                .field("date", "2015-01-01T12:10:30.123456789Z")
+                .field("date_nanos", "2015-01-01T12:10:30.123456789Z")
+                .endObject();
+            index.addParameter("refresh", "true");
+            index.setJsonEntity(Strings.toString(doc));
+            client().performRequest(index);
+        } else if (isUpgradedCluster()) {
+            Request search = new Request("POST", "/" + indexName + "/_search");
+            XContentBuilder query = XContentBuilder.builder(XContentType.JSON.xContent())
+                .startObject()
+                .array("fields", new String[] { "date", "date_nanos" })
+                .endObject();
+            search.setJsonEntity(Strings.toString(query));
+            Map<String, Object> response = entityAsMap(client().performRequest(search));
+            Map<?, ?> bestHit = (Map<?, ?>) ((List<?>) (XContentMapValues.extractValue("hits.hits", response))).get(0);
+            List<?> date = (List<?>) XContentMapValues.extractValue("fields.date", bestHit);
+            assertThat(date.size(), equalTo(1));
+            assertThat(date.get(0), equalTo("2015-01-01T12:10:30.123Z"));
+            List<?> dateNanos = (List<?>) XContentMapValues.extractValue("fields.date_nanos", bestHit);
+            assertThat(dateNanos.size(), equalTo(1));
+            assertThat(dateNanos.get(0), equalTo("2015-01-01T12:10:30.123456789Z"));
         }
     }
 
@@ -247,51 +229,45 @@ public class IndexingIT extends AbstractRollingTestCase {
     }
 
     public void testTsdb() throws IOException {
-        assumeTrue("indexing time series indices changed in 8.2.0", UPGRADE_FROM_VERSION.onOrAfter(Version.V_8_2_0));
+        assumeTrue("indexing time series indices changed in 8.2.0", getOldClusterVersion().onOrAfter(Version.V_8_2_0));
 
         StringBuilder bulk = new StringBuilder();
-        switch (CLUSTER_TYPE) {
-            case OLD -> {
-                createTsdbIndex();
-                tsdbBulk(bulk, TSDB_DIMS.get(0), TSDB_TIMES[0], TSDB_TIMES[1], 0.1);
-                tsdbBulk(bulk, TSDB_DIMS.get(1), TSDB_TIMES[0], TSDB_TIMES[1], -0.1);
-                bulk("tsdb", bulk.toString());
-                assertTsdbAgg(closeTo(215.95, 0.005), closeTo(-215.95, 0.005));
-                return;
-            }
-            case MIXED -> {
-                if (FIRST_MIXED_ROUND) {
-                    tsdbBulk(bulk, TSDB_DIMS.get(0), TSDB_TIMES[1], TSDB_TIMES[2], 0.1);
-                    tsdbBulk(bulk, TSDB_DIMS.get(1), TSDB_TIMES[1], TSDB_TIMES[2], -0.1);
-                    tsdbBulk(bulk, TSDB_DIMS.get(2), TSDB_TIMES[0], TSDB_TIMES[2], 1.1);
-                    bulk("tsdb", bulk.toString());
-                    assertTsdbAgg(closeTo(217.45, 0.005), closeTo(-217.45, 0.005), closeTo(2391.95, 0.005));
-                    return;
-                }
-                tsdbBulk(bulk, TSDB_DIMS.get(0), TSDB_TIMES[2], TSDB_TIMES[3], 0.1);
-                tsdbBulk(bulk, TSDB_DIMS.get(1), TSDB_TIMES[2], TSDB_TIMES[3], -0.1);
-                tsdbBulk(bulk, TSDB_DIMS.get(2), TSDB_TIMES[2], TSDB_TIMES[3], 1.1);
-                tsdbBulk(bulk, TSDB_DIMS.get(3), TSDB_TIMES[0], TSDB_TIMES[3], 10);
-                bulk("tsdb", bulk.toString());
-                assertTsdbAgg(closeTo(218.95, 0.005), closeTo(-218.95, 0.005), closeTo(2408.45, 0.005), closeTo(21895, 0.5));
-                return;
-            }
-            case UPGRADED -> {
-                tsdbBulk(bulk, TSDB_DIMS.get(0), TSDB_TIMES[3], TSDB_TIMES[4], 0.1);
-                tsdbBulk(bulk, TSDB_DIMS.get(1), TSDB_TIMES[3], TSDB_TIMES[4], -0.1);
-                tsdbBulk(bulk, TSDB_DIMS.get(2), TSDB_TIMES[3], TSDB_TIMES[4], 1.1);
-                tsdbBulk(bulk, TSDB_DIMS.get(3), TSDB_TIMES[3], TSDB_TIMES[4], 10);
-                tsdbBulk(bulk, TSDB_DIMS.get(4), TSDB_TIMES[0], TSDB_TIMES[4], -5);
-                bulk("tsdb", bulk.toString());
-                assertTsdbAgg(
-                    closeTo(220.45, 0.005),
-                    closeTo(-220.45, 0.005),
-                    closeTo(2424.95, 0.005),
-                    closeTo(22045, 0.5),
-                    closeTo(-11022.5, 0.5)
-                );
-                return;
-            }
+        if (isOldCluster()) {
+            createTsdbIndex();
+            tsdbBulk(bulk, TSDB_DIMS.get(0), TSDB_TIMES[0], TSDB_TIMES[1], 0.1);
+            tsdbBulk(bulk, TSDB_DIMS.get(1), TSDB_TIMES[0], TSDB_TIMES[1], -0.1);
+            bulk("tsdb", bulk.toString());
+            assertTsdbAgg(closeTo(215.95, 0.005), closeTo(-215.95, 0.005));
+            return;
+        } else if (isFirstMixedCluster()) {
+            tsdbBulk(bulk, TSDB_DIMS.get(0), TSDB_TIMES[1], TSDB_TIMES[2], 0.1);
+            tsdbBulk(bulk, TSDB_DIMS.get(1), TSDB_TIMES[1], TSDB_TIMES[2], -0.1);
+            tsdbBulk(bulk, TSDB_DIMS.get(2), TSDB_TIMES[0], TSDB_TIMES[2], 1.1);
+            bulk("tsdb", bulk.toString());
+            assertTsdbAgg(closeTo(217.45, 0.005), closeTo(-217.45, 0.005), closeTo(2391.95, 0.005));
+
+        } else if (isMixedCluster()) {
+            tsdbBulk(bulk, TSDB_DIMS.get(0), TSDB_TIMES[2], TSDB_TIMES[3], 0.1);
+            tsdbBulk(bulk, TSDB_DIMS.get(1), TSDB_TIMES[2], TSDB_TIMES[3], -0.1);
+            tsdbBulk(bulk, TSDB_DIMS.get(2), TSDB_TIMES[2], TSDB_TIMES[3], 1.1);
+            tsdbBulk(bulk, TSDB_DIMS.get(3), TSDB_TIMES[0], TSDB_TIMES[3], 10);
+            bulk("tsdb", bulk.toString());
+            assertTsdbAgg(closeTo(218.95, 0.005), closeTo(-218.95, 0.005), closeTo(2408.45, 0.005), closeTo(21895, 0.5));
+            return;
+        } else {
+            tsdbBulk(bulk, TSDB_DIMS.get(0), TSDB_TIMES[3], TSDB_TIMES[4], 0.1);
+            tsdbBulk(bulk, TSDB_DIMS.get(1), TSDB_TIMES[3], TSDB_TIMES[4], -0.1);
+            tsdbBulk(bulk, TSDB_DIMS.get(2), TSDB_TIMES[3], TSDB_TIMES[4], 1.1);
+            tsdbBulk(bulk, TSDB_DIMS.get(3), TSDB_TIMES[3], TSDB_TIMES[4], 10);
+            tsdbBulk(bulk, TSDB_DIMS.get(4), TSDB_TIMES[0], TSDB_TIMES[4], -5);
+            bulk("tsdb", bulk.toString());
+            assertTsdbAgg(
+                closeTo(220.45, 0.005),
+                closeTo(-220.45, 0.005),
+                closeTo(2424.95, 0.005),
+                closeTo(22045, 0.5),
+                closeTo(-11022.5, 0.5)
+            );
         }
     }
 
@@ -361,67 +337,60 @@ public class IndexingIT extends AbstractRollingTestCase {
     }
 
     public void testSyntheticSource() throws IOException {
-        assumeTrue("added in 8.4.0", UPGRADE_FROM_VERSION.onOrAfter(Version.V_8_4_0));
-
-        switch (CLUSTER_TYPE) {
-            case OLD -> {
-                Request createIndex = new Request("PUT", "/synthetic");
-                XContentBuilder indexSpec = XContentBuilder.builder(XContentType.JSON.xContent()).startObject();
-                indexSpec.startObject("mappings");
-                {
-                    indexSpec.startObject("_source").field("mode", "synthetic").endObject();
-                    indexSpec.startObject("properties").startObject("kwd").field("type", "keyword").endObject().endObject();
-                }
-                indexSpec.endObject();
-                createIndex.setJsonEntity(Strings.toString(indexSpec.endObject()));
-                client().performRequest(createIndex);
-                bulk("synthetic", """
-                    {"index": {"_index": "synthetic", "_id": "old"}}
-                    {"kwd": "old", "int": -12}
-                    """);
-                break;
-            }
-            case MIXED -> {
-                if (FIRST_MIXED_ROUND) {
-                    bulk("synthetic", """
-                        {"index": {"_index": "synthetic", "_id": "mixed_1"}}
-                        {"kwd": "mixed_1", "int": 22}
-                        """);
-                } else {
-                    bulk("synthetic", """
-                        {"index": {"_index": "synthetic", "_id": "mixed_2"}}
-                        {"kwd": "mixed_2", "int": 33}
-                        """);
-                }
-                break;
-            }
-            case UPGRADED -> {
-                bulk("synthetic", """
-                    {"index": {"_index": "synthetic", "_id": "new"}}
-                    {"kwd": "new", "int": 21341325}
-                    """);
+        assumeTrue("added in 8.4.0", getOldClusterVersion().onOrAfter(Version.V_8_4_0));
+
+        if (isOldCluster()) {
+            Request createIndex = new Request("PUT", "/synthetic");
+            XContentBuilder indexSpec = XContentBuilder.builder(XContentType.JSON.xContent()).startObject();
+            indexSpec.startObject("mappings");
+            {
+                indexSpec.startObject("_source").field("mode", "synthetic").endObject();
+                indexSpec.startObject("properties").startObject("kwd").field("type", "keyword").endObject().endObject();
             }
+            indexSpec.endObject();
+            createIndex.setJsonEntity(Strings.toString(indexSpec.endObject()));
+            client().performRequest(createIndex);
+            bulk("synthetic", """
+                {"index": {"_index": "synthetic", "_id": "old"}}
+                {"kwd": "old", "int": -12}
+                """);
+        } else if (isFirstMixedCluster()) {
+            bulk("synthetic", """
+                {"index": {"_index": "synthetic", "_id": "mixed_1"}}
+                {"kwd": "mixed_1", "int": 22}
+                """);
+        } else if (isMixedCluster()) {
+            bulk("synthetic", """
+                {"index": {"_index": "synthetic", "_id": "mixed_2"}}
+                {"kwd": "mixed_2", "int": 33}
+                """);
+
+        } else {
+            bulk("synthetic", """
+                {"index": {"_index": "synthetic", "_id": "new"}}
+                {"kwd": "new", "int": 21341325}
+                """);
         }
 
         assertMap(
             entityAsMap(client().performRequest(new Request("GET", "/synthetic/_doc/old"))),
             matchesMap().extraOk().entry("_source", matchesMap().entry("kwd", "old").entry("int", -12))
         );
-        if (CLUSTER_TYPE == ClusterType.OLD) {
+        if (isOldCluster()) {
             return;
         }
         assertMap(
             entityAsMap(client().performRequest(new Request("GET", "/synthetic/_doc/mixed_1"))),
             matchesMap().extraOk().entry("_source", matchesMap().entry("kwd", "mixed_1").entry("int", 22))
         );
-        if (CLUSTER_TYPE == ClusterType.MIXED && FIRST_MIXED_ROUND) {
+        if (isFirstMixedCluster()) {
             return;
         }
         assertMap(
             entityAsMap(client().performRequest(new Request("GET", "/synthetic/_doc/mixed_2"))),
             matchesMap().extraOk().entry("_source", matchesMap().entry("kwd", "mixed_2").entry("int", 33))
         );
-        if (CLUSTER_TYPE == ClusterType.MIXED) {
+        if (isMixedCluster()) {
             return;
         }
         assertMap(

qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedRollingUpgradeTestCase.java

@@ -0,0 +1,226 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.upgrades;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.FeatureFlag;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.elasticsearch.test.cluster.util.Version;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.test.rest.ObjectPath;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TemporaryFolder;
+import org.junit.rules.TestRule;
+
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Supplier;
+import java.util.stream.IntStream;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
+
+public abstract class ParameterizedRollingUpgradeTestCase extends ESRestTestCase {
+    private static final Version OLD_CLUSTER_VERSION = Version.fromString(System.getProperty("tests.old_cluster_version"));
+
+    private static final TemporaryFolder repoDirectory = new TemporaryFolder();
+
+    private static final int NODE_NUM = 3;
+
+    private static final ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .distribution(DistributionType.DEFAULT)
+        .version(getOldClusterTestVersion())
+        .nodes(NODE_NUM)
+        .setting("path.repo", new Supplier<>() {
+            @Override
+            @SuppressForbidden(reason = "TemporaryFolder only has io.File methods, not nio.File")
+            public String get() {
+                return repoDirectory.getRoot().getPath();
+            }
+        })
+        .setting("xpack.security.enabled", "false")
+        .feature(FeatureFlag.TIME_SERIES_MODE)
+        .build();
+
+    @ClassRule
+    public static TestRule ruleChain = RuleChain.outerRule(repoDirectory).around(cluster);
+
+    @ParametersFactory(shuffle = false)
+    public static Iterable<Object[]> parameters() {
+        return IntStream.rangeClosed(0, NODE_NUM).boxed().map(n -> new Object[] { n }).toList();
+    }
+
+    private static final Set<Integer> upgradedNodes = new HashSet<>();
+    private static boolean upgradeFailed = false;
+    private static IndexVersion oldIndexVersion;
+
+    private final int requestedUpgradedNodes;
+
+    protected ParameterizedRollingUpgradeTestCase(@Name("upgradedNodes") int upgradedNodes) {
+        this.requestedUpgradedNodes = upgradedNodes;
+    }
+
+    @Before
+    public void extractOldIndexVersion() throws Exception {
+        if (oldIndexVersion == null && upgradedNodes.isEmpty()) {
+            IndexVersion indexVersion = null;   // these should all be the same version
+
+            Request request = new Request("GET", "_nodes");
+            request.addParameter("filter_path", "nodes.*.index_version,nodes.*.name");
+            Response response = client().performRequest(request);
+            ObjectPath objectPath = ObjectPath.createFromResponse(response);
+            Map<String, Object> nodeMap = objectPath.evaluate("nodes");
+            for (String id : nodeMap.keySet()) {
+                Number ix = objectPath.evaluate("nodes." + id + ".index_version");
+                IndexVersion version;
+                if (ix != null) {
+                    version = IndexVersion.fromId(ix.intValue());
+                } else {
+                    // it doesn't have index version (pre 8.11) - just infer it from the release version
+                    version = IndexVersion.fromId(getOldClusterVersion().id);
+                }
+
+                if (indexVersion == null) {
+                    indexVersion = version;
+                } else {
+                    String name = objectPath.evaluate("nodes." + id + ".name");
+                    assertThat("Node " + name + " has a different index version to other nodes", version, equalTo(indexVersion));
+                }
+            }
+
+            assertThat("Index version could not be read", indexVersion, notNullValue());
+            oldIndexVersion = indexVersion;
+        }
+    }
+
+    @Before
+    public void upgradeNode() throws Exception {
+        // Skip remaining tests if upgrade failed
+        assumeFalse("Cluster upgrade failed", upgradeFailed);
+
+        if (upgradedNodes.size() < requestedUpgradedNodes) {
+            closeClients();
+            // we might be running a specific upgrade test by itself - check previous nodes too
+            for (int n = 0; n < requestedUpgradedNodes; n++) {
+                if (upgradedNodes.add(n)) {
+                    try {
+                        logger.info("Upgrading node {} to version {}", n, Version.CURRENT);
+                        cluster.upgradeNodeToVersion(n, Version.CURRENT);
+                    } catch (Exception e) {
+                        upgradeFailed = true;
+                        throw e;
+                    }
+                }
+            }
+            initClient();
+        }
+    }
+
+    @AfterClass
+    public static void resetNodes() {
+        oldIndexVersion = null;
+        upgradedNodes.clear();
+        upgradeFailed = false;
+    }
+
+    protected static org.elasticsearch.Version getOldClusterVersion() {
+        return org.elasticsearch.Version.fromString(OLD_CLUSTER_VERSION.toString());
+    }
+
+    protected static IndexVersion getOldClusterIndexVersion() {
+        assert oldIndexVersion != null;
+        return oldIndexVersion;
+    }
+
+    protected static Version getOldClusterTestVersion() {
+        return Version.fromString(OLD_CLUSTER_VERSION.toString());
+    }
+
+    protected static boolean isOldCluster() {
+        return upgradedNodes.isEmpty();
+    }
+
+    protected static boolean isFirstMixedCluster() {
+        return upgradedNodes.size() == 1;
+    }
+
+    protected static boolean isMixedCluster() {
+        return upgradedNodes.isEmpty() == false && upgradedNodes.size() < NODE_NUM;
+    }
+
+    protected static boolean isUpgradedCluster() {
+        return upgradedNodes.size() == NODE_NUM;
+    }
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
+    @Override
+    protected final boolean resetFeatureStates() {
+        return false;
+    }
+
+    @Override
+    protected final boolean preserveIndicesUponCompletion() {
+        return true;
+    }
+
+    @Override
+    protected final boolean preserveDataStreamsUponCompletion() {
+        return true;
+    }
+
+    @Override
+    protected final boolean preserveReposUponCompletion() {
+        return true;
+    }
+
+    @Override
+    protected boolean preserveTemplatesUponCompletion() {
+        return true;
+    }
+
+    @Override
+    protected boolean preserveClusterUponCompletion() {
+        return true;
+    }
+
+    @Override
+    protected final Settings restClientSettings() {
+        return Settings.builder()
+            .put(super.restClientSettings())
+            // increase the timeout here to 90 seconds to handle long waits for a green
+            // cluster health. the waits for green need to be longer than a minute to
+            // account for delayed shards
+            .put(ESRestTestCase.CLIENT_SOCKET_TIMEOUT, "90s")
+            .build();
+    }
+
+    @Override
+    protected final String getEnsureGreenTimeout() {
+        // increase the timeout here to 70 seconds to handle long waits for a green
+        // cluster health. the waits for green need to be longer than a minute to
+        // account for delayed shards
+        return "70s";
+    }
+}

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SnapshotBasedRecoveryIT.java → qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SnapshotBasedRecoveryIT.java

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.upgrades;
 
+import com.carrotsearch.randomizedtesting.annotations.Name;
+
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.util.EntityUtils;
@@ -40,100 +42,102 @@ import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.lessThan;
 import static org.hamcrest.Matchers.notNullValue;
 
-public class SnapshotBasedRecoveryIT extends AbstractRollingTestCase {
+public class SnapshotBasedRecoveryIT extends ParameterizedRollingUpgradeTestCase {
+
+    public SnapshotBasedRecoveryIT(@Name("upgradedNodes") int upgradedNodes) {
+        super(upgradedNodes);
+    }
 
     public void testSnapshotBasedRecovery() throws Exception {
 
         assumeFalse(
             "Cancel shard allocation command is broken for initial desired balance versions and might allocate shard "
                 + "on the node where it is not supposed to be. Fixed by https://github.com/elastic/elasticsearch/pull/93635",
-            UPGRADE_FROM_VERSION == Version.V_8_6_0 || UPGRADE_FROM_VERSION == Version.V_8_6_1 || UPGRADE_FROM_VERSION == Version.V_8_7_0
+            getOldClusterVersion() == Version.V_8_6_0
+                || getOldClusterVersion() == Version.V_8_6_1
+                || getOldClusterVersion() == Version.V_8_7_0
         );
 
         final String indexName = "snapshot_based_recovery";
         final String repositoryName = "snapshot_based_recovery_repo";
         final int numDocs = 200;
-        switch (CLUSTER_TYPE) {
-            case OLD -> {
-                Settings.Builder settings = Settings.builder()
-                    .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-                    .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
-                    .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms")
-                    .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster
-                createIndex(indexName, settings.build());
-                ensureGreen(indexName);
-                indexDocs(indexName, numDocs);
-                flush(indexName, true);
-                registerRepository(
-                    repositoryName,
-                    "fs",
-                    true,
-                    Settings.builder()
-                        .put("location", "./snapshot_based_recovery")
-                        .put(BlobStoreRepository.USE_FOR_PEER_RECOVERY_SETTING.getKey(), true)
-                        .build()
-                );
-                createSnapshot(repositoryName, "snap", true);
-                updateIndexSettings(indexName, Settings.builder().put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1));
-                ensureGreen(indexName);
-            }
-            case MIXED, UPGRADED -> {
-                if (FIRST_MIXED_ROUND) {
-                    List<String> upgradedNodeIds = getUpgradedNodeIds();
-                    // It's possible that the test simply does a rolling-restart, i.e. it "upgrades" to
-                    // the same version. In that case we proceed without excluding any node
-                    if (upgradedNodeIds.isEmpty() == false) {
-                        assertThat(upgradedNodeIds.size(), is(equalTo(1)));
-                        String upgradedNodeId = upgradedNodeIds.get(0);
-                        logger.info("--> excluding [{}] from node [{}]", indexName, upgradedNodeId);
-                        updateIndexSettings(indexName, Settings.builder().put("index.routing.allocation.exclude._id", upgradedNodeId));
-                        ensureGreen(indexName);
-                        logger.info("--> finished excluding [{}] from node [{}]", indexName, upgradedNodeId);
-                    } else {
-                        logger.info("--> no upgrading nodes, not adding any exclusions for [{}]", indexName);
-                    }
+        if (isOldCluster()) {
+            Settings.Builder settings = Settings.builder()
+                .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
+                .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
+                .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms")
+                .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster
+            createIndex(indexName, settings.build());
+            ensureGreen(indexName);
+            indexDocs(indexName, numDocs);
+            flush(indexName, true);
+            registerRepository(
+                repositoryName,
+                "fs",
+                true,
+                Settings.builder()
+                    .put("location", "./snapshot_based_recovery")
+                    .put(BlobStoreRepository.USE_FOR_PEER_RECOVERY_SETTING.getKey(), true)
+                    .build()
+            );
+            createSnapshot(repositoryName, "snap", true);
+            updateIndexSettings(indexName, Settings.builder().put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1));
+            ensureGreen(indexName);
+        } else {
+            if (isFirstMixedCluster()) {
+                List<String> upgradedNodeIds = getUpgradedNodeIds();
+                // It's possible that the test simply does a rolling-restart, i.e. it "upgrades" to
+                // the same version. In that case we proceed without excluding any node
+                if (upgradedNodeIds.isEmpty() == false) {
+                    assertThat(upgradedNodeIds.size(), is(equalTo(1)));
+                    String upgradedNodeId = upgradedNodeIds.get(0);
+                    logger.info("--> excluding [{}] from node [{}]", indexName, upgradedNodeId);
+                    updateIndexSettings(indexName, Settings.builder().put("index.routing.allocation.exclude._id", upgradedNodeId));
+                    ensureGreen(indexName);
+                    logger.info("--> finished excluding [{}] from node [{}]", indexName, upgradedNodeId);
+                } else {
+                    logger.info("--> no upgrading nodes, not adding any exclusions for [{}]", indexName);
+                }
 
-                    String primaryNodeId = getPrimaryNodeIdOfShard(indexName, 0);
-                    Version primaryNodeVersion = getNodeVersion(primaryNodeId);
+                String primaryNodeId = getPrimaryNodeIdOfShard(indexName, 0);
+                Version primaryNodeVersion = getNodeVersion(primaryNodeId);
 
-                    // Sometimes the primary shard ends on the upgraded node (i.e. after a rebalance)
-                    // This causes issues when removing and adding replicas, since then we cannot allocate to any of the old nodes.
-                    // That is an issue only for the first mixed round.
-                    // In that case we exclude the upgraded node from the shard allocation and cancel the shard to force moving
-                    // the primary to a node in the old version, this allows adding replicas in the first mixed round.
-                    logger.info("--> Primary node in first mixed round {} / {}", primaryNodeId, primaryNodeVersion);
-                    if (primaryNodeVersion.after(UPGRADE_FROM_VERSION)) {
-                        logger.info("--> cancelling primary shard on node [{}]", primaryNodeId);
-                        cancelShard(indexName, 0, primaryNodeId);
-                        logger.info("--> done cancelling primary shard on node [{}]", primaryNodeId);
+                // Sometimes the primary shard ends on the upgraded node (i.e. after a rebalance)
+                // This causes issues when removing and adding replicas, since then we cannot allocate to any of the old nodes.
+                // That is an issue only for the first mixed round.
+                // In that case we exclude the upgraded node from the shard allocation and cancel the shard to force moving
+                // the primary to a node in the old version, this allows adding replicas in the first mixed round.
+                logger.info("--> Primary node in first mixed round {} / {}", primaryNodeId, primaryNodeVersion);
+                if (primaryNodeVersion.after(getOldClusterVersion())) {
+                    logger.info("--> cancelling primary shard on node [{}]", primaryNodeId);
+                    cancelShard(indexName, 0, primaryNodeId);
+                    logger.info("--> done cancelling primary shard on node [{}]", primaryNodeId);
 
-                        String currentPrimaryNodeId = getPrimaryNodeIdOfShard(indexName, 0);
-                        assertThat(getNodeVersion(currentPrimaryNodeId), is(equalTo(UPGRADE_FROM_VERSION)));
-                    }
-                } else {
-                    logger.info("--> not in first upgrade round, removing exclusions for [{}]", indexName);
-                    updateIndexSettings(indexName, Settings.builder().putNull("index.routing.allocation.exclude._id"));
-                    logger.info("--> done removing exclusions for [{}]", indexName);
+                    String currentPrimaryNodeId = getPrimaryNodeIdOfShard(indexName, 0);
+                    assertThat(getNodeVersion(currentPrimaryNodeId), is(equalTo(getOldClusterVersion())));
                 }
+            } else {
+                logger.info("--> not in first upgrade round, removing exclusions for [{}]", indexName);
+                updateIndexSettings(indexName, Settings.builder().putNull("index.routing.allocation.exclude._id"));
+                logger.info("--> done removing exclusions for [{}]", indexName);
+            }
 
-                // Drop replicas
-                logger.info("--> dropping replicas from [{}]", indexName);
-                updateIndexSettingsPermittingSlowlogDeprecationWarning(
-                    indexName,
-                    Settings.builder().put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
-                );
-                logger.info("--> finished dropping replicas from [{}], adding them back", indexName);
-                updateIndexSettingsPermittingSlowlogDeprecationWarning(
-                    indexName,
-                    Settings.builder().put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)
-                );
-                logger.info("--> finished adding replicas from [{}]", indexName);
-                ensureGreen(indexName);
+            // Drop replicas
+            logger.info("--> dropping replicas from [{}]", indexName);
+            updateIndexSettingsPermittingSlowlogDeprecationWarning(
+                indexName,
+                Settings.builder().put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
+            );
+            logger.info("--> finished dropping replicas from [{}], adding them back", indexName);
+            updateIndexSettingsPermittingSlowlogDeprecationWarning(
+                indexName,
+                Settings.builder().put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)
+            );
+            logger.info("--> finished adding replicas from [{}]", indexName);
+            ensureGreen(indexName);
 
-                assertMatchAllReturnsAllDocuments(indexName, numDocs);
-                assertMatchQueryReturnsAllDocuments(indexName, numDocs);
-            }
-            default -> throw new IllegalStateException("unknown type " + CLUSTER_TYPE);
+            assertMatchAllReturnsAllDocuments(indexName, numDocs);
+            assertMatchQueryReturnsAllDocuments(indexName, numDocs);
         }
     }
 
@@ -145,7 +149,7 @@ public class SnapshotBasedRecoveryIT extends AbstractRollingTestCase {
         List<String> upgradedNodes = new ArrayList<>();
         for (Map.Entry<String, Map<String, Object>> nodeInfoEntry : nodes.entrySet()) {
             Version nodeVersion = Version.fromString(extractValue(nodeInfoEntry.getValue(), "version"));
-            if (nodeVersion.after(UPGRADE_FROM_VERSION)) {
+            if (nodeVersion.after(getOldClusterVersion())) {
                 upgradedNodes.add(nodeInfoEntry.getKey());
             }
         }

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SystemIndicesUpgradeIT.java → qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SystemIndicesUpgradeIT.java

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.upgrades;
 
+import com.carrotsearch.randomizedtesting.annotations.Name;
+
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.index.IndexVersion;
@@ -21,13 +23,17 @@ import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
 
-public class SystemIndicesUpgradeIT extends AbstractRollingTestCase {
+public class SystemIndicesUpgradeIT extends ParameterizedRollingUpgradeTestCase {
+
+    public SystemIndicesUpgradeIT(@Name("upgradedNodes") int upgradedNodes) {
+        super(upgradedNodes);
+    }
 
     @SuppressWarnings("unchecked")
     public void testSystemIndicesUpgrades() throws Exception {
         final String systemIndexWarning = "this request accesses system indices: [.tasks], but in a future major version, direct "
             + "access to system indices will be prevented by default";
-        if (CLUSTER_TYPE == ClusterType.OLD) {
+        if (isOldCluster()) {
             // create index
             Request createTestIndex = new Request("PUT", "/test_index_old");
             createTestIndex.setJsonEntity("{\"settings\": {\"index.number_of_replicas\": 0}}");
@@ -99,7 +105,7 @@ public class SystemIndicesUpgradeIT extends AbstractRollingTestCase {
                 }));
                 assertThat(client().performRequest(putAliasRequest).getStatusLine().getStatusCode(), is(200));
             }
-        } else if (CLUSTER_TYPE == ClusterType.UPGRADED) {
+        } else if (isUpgradedCluster()) {
             assertBusy(() -> {
                 Request clusterStateRequest = new Request("GET", "/_cluster/state/metadata");
                 Map<String, Object> indices = new JsonMapView(entityAsMap(client().performRequest(clusterStateRequest))).get(

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TsdbIT.java → qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/TsdbIT.java

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.upgrades;
 
+import com.carrotsearch.randomizedtesting.annotations.Name;
+
 import org.elasticsearch.Version;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.common.time.DateFormatter;
@@ -24,7 +26,11 @@ import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
-public class TsdbIT extends AbstractRollingTestCase {
+public class TsdbIT extends ParameterizedRollingUpgradeTestCase {
+
+    public TsdbIT(@Name("upgradedNodes") int upgradedNodes) {
+        super(upgradedNodes);
+    }
 
     private static final String TEMPLATE = """
         {
@@ -88,21 +94,21 @@ public class TsdbIT extends AbstractRollingTestCase {
     private static final String BULK =
         """
             {"create": {}}
-            {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507","ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}
+            {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}
             {"create": {}}
-            {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "hamster", "uid":"947e4ced-1786-4e53-9e0c-5c447e959508","ip": "10.10.55.1", "network": {"tx": 2005177954, "rx": 801479970}}}}
+            {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "hamster", "uid":"947e4ced-1786-4e53-9e0c-5c447e959508", "ip": "10.10.55.1", "network": {"tx": 2005177954, "rx": 801479970}}}}
             {"create": {}}
-            {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "cow", "uid":"947e4ced-1786-4e53-9e0c-5c447e959509","ip": "10.10.55.1", "network": {"tx": 2006223737, "rx": 802337279}}}}
+            {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "cow", "uid":"947e4ced-1786-4e53-9e0c-5c447e959509", "ip": "10.10.55.1", "network": {"tx": 2006223737, "rx": 802337279}}}}
             {"create": {}}
-            {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "rat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959510","ip": "10.10.55.2", "network": {"tx": 2012916202, "rx": 803685721}}}}
+            {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "rat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959510", "ip": "10.10.55.2", "network": {"tx": 2012916202, "rx": 803685721}}}}
             {"create": {}}
-            {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9","ip": "10.10.55.3", "network": {"tx": 1434521831, "rx": 530575198}}}}
+            {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434521831, "rx": 530575198}}}}
             {"create": {}}
-            {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "tiger", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea10","ip": "10.10.55.3", "network": {"tx": 1434577921, "rx": 530600088}}}}
+            {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "tiger", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea10", "ip": "10.10.55.3", "network": {"tx": 1434577921, "rx": 530600088}}}}
             {"create": {}}
-            {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "lion", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876e11","ip": "10.10.55.3", "network": {"tx": 1434587694, "rx": 530604797}}}}
+            {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "lion", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876e11", "ip": "10.10.55.3", "network": {"tx": 1434587694, "rx": 530604797}}}}
             {"create": {}}
-            {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "elephant", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876eb4","ip": "10.10.55.3", "network": {"tx": 1434595272, "rx": 530605511}}}}
+            {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "elephant", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876eb4", "ip": "10.10.55.3", "network": {"tx": 1434595272, "rx": 530605511}}}}
             """;
 
     private static final String DOC = """
@@ -125,11 +131,11 @@ public class TsdbIT extends AbstractRollingTestCase {
 
     public void testTsdbDataStream() throws Exception {
         assumeTrue(
-            "Skipping version [" + UPGRADE_FROM_VERSION + "], because TSDB was GA-ed in 8.7.0",
-            UPGRADE_FROM_VERSION.onOrAfter(Version.V_8_7_0)
+            "Skipping version [" + getOldClusterVersion() + "], because TSDB was GA-ed in 8.7.0",
+            getOldClusterVersion().onOrAfter(Version.V_8_7_0)
         );
         String dataStreamName = "k8s";
-        if (CLUSTER_TYPE == ClusterType.OLD) {
+        if (isOldCluster()) {
             final String INDEX_TEMPLATE = """
                 {
                     "index_patterns": ["$PATTERN"],
@@ -144,20 +150,20 @@ public class TsdbIT extends AbstractRollingTestCase {
             assertOK(client().performRequest(putIndexTemplateRequest));
 
             performOldClustertOperations(templateName, dataStreamName);
-        } else if (CLUSTER_TYPE == ClusterType.MIXED) {
+        } else if (isMixedCluster()) {
             performMixedClusterOperations(dataStreamName);
-        } else if (CLUSTER_TYPE == ClusterType.UPGRADED) {
+        } else if (isUpgradedCluster()) {
             performUpgradedClusterOperations(dataStreamName);
         }
     }
 
     public void testTsdbDataStreamWithComponentTemplate() throws Exception {
         assumeTrue(
-            "Skipping version [" + UPGRADE_FROM_VERSION + "], because TSDB was GA-ed in 8.7.0 and bug was fixed in 8.11.0",
-            UPGRADE_FROM_VERSION.onOrAfter(Version.V_8_7_0) && UPGRADE_FROM_VERSION.before(Version.V_8_11_0)
+            "Skipping version [" + getOldClusterVersion() + "], because TSDB was GA-ed in 8.7.0 and bug was fixed in 8.11.0",
+            getOldClusterVersion().onOrAfter(Version.V_8_7_0) && getOldClusterVersion().before(Version.V_8_11_0)
         );
         String dataStreamName = "template-with-component-template";
-        if (CLUSTER_TYPE == ClusterType.OLD) {
+        if (isOldCluster()) {
             final String COMPONENT_TEMPLATE = """
                     {
                         "template": $TEMPLATE
@@ -181,9 +187,9 @@ public class TsdbIT extends AbstractRollingTestCase {
             assertOK(client().performRequest(putIndexTemplateRequest));
 
             performOldClustertOperations(templateName, dataStreamName);
-        } else if (CLUSTER_TYPE == ClusterType.MIXED) {
+        } else if (isMixedCluster()) {
             performMixedClusterOperations(dataStreamName);
-        } else if (CLUSTER_TYPE == ClusterType.UPGRADED) {
+        } else if (isUpgradedCluster()) {
             performUpgradedClusterOperations(dataStreamName);
 
             var dataStreams = getDataStream(dataStreamName);
@@ -242,7 +248,7 @@ public class TsdbIT extends AbstractRollingTestCase {
 
     private static void performMixedClusterOperations(String dataStreamName) throws IOException {
         ensureHealth(dataStreamName, request -> request.addParameter("wait_for_status", "yellow"));
-        if (FIRST_MIXED_ROUND) {
+        if (isFirstMixedCluster()) {
             indexDoc(dataStreamName);
         }
         assertSearch(dataStreamName, 9);

qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/UpgradeWithOldIndexSettingsIT.java

@@ -0,0 +1,132 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.upgrades;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.core.Strings;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static org.elasticsearch.rest.action.search.RestSearchAction.TOTAL_HITS_AS_INT_PARAM;
+import static org.hamcrest.Matchers.is;
+
+public class UpgradeWithOldIndexSettingsIT extends ParameterizedRollingUpgradeTestCase {
+
+    public UpgradeWithOldIndexSettingsIT(@Name("upgradedNodes") int upgradedNodes) {
+        super(upgradedNodes);
+    }
+
+    private static final String INDEX_NAME = "test_index_old_settings";
+    private static final String EXPECTED_WARNING = "[index.indexing.slowlog.level] setting was deprecated in Elasticsearch and will "
+        + "be removed in a future release! See the breaking changes documentation for the next major version.";
+
+    private static final String EXPECTED_V8_WARNING = "[index.indexing.slowlog.level] setting was deprecated in the previous Elasticsearch"
+        + " release and is removed in this release.";
+
+    public void testOldIndexSettings() throws Exception {
+        if (isOldCluster()) {
+            Request createTestIndex = new Request("PUT", "/" + INDEX_NAME);
+            createTestIndex.setJsonEntity("{\"settings\": {\"index.indexing.slowlog.level\": \"WARN\"}}");
+            createTestIndex.setOptions(expectWarnings(EXPECTED_WARNING));
+            if (getOldClusterVersion().before(Version.V_8_0_0)) {
+                // create index with settings no longer valid in 8.0
+                client().performRequest(createTestIndex);
+            } else {
+                assertTrue(
+                    expectThrows(ResponseException.class, () -> client().performRequest(createTestIndex)).getMessage()
+                        .contains("unknown setting [index.indexing.slowlog.level]")
+                );
+
+                Request createTestIndex1 = new Request("PUT", "/" + INDEX_NAME);
+                client().performRequest(createTestIndex1);
+            }
+
+            // add some data
+            Request bulk = new Request("POST", "/_bulk");
+            bulk.addParameter("refresh", "true");
+            if (getOldClusterVersion().before(Version.V_8_0_0)) {
+                bulk.setOptions(expectWarnings(EXPECTED_WARNING));
+            }
+            bulk.setJsonEntity(Strings.format("""
+                {"index": {"_index": "%s"}}
+                {"f1": "v1", "f2": "v2"}
+                """, INDEX_NAME));
+            client().performRequest(bulk);
+        } else if (isMixedCluster()) {
+            // add some more data
+            Request bulk = new Request("POST", "/_bulk");
+            bulk.addParameter("refresh", "true");
+            if (getOldClusterVersion().before(Version.V_8_0_0)) {
+                bulk.setOptions(expectWarnings(EXPECTED_WARNING));
+            }
+            bulk.setJsonEntity(Strings.format("""
+                {"index": {"_index": "%s"}}
+                {"f1": "v3", "f2": "v4"}
+                """, INDEX_NAME));
+            client().performRequest(bulk);
+        } else {
+            if (getOldClusterVersion().before(Version.V_8_0_0)) {
+                Request createTestIndex = new Request("PUT", "/" + INDEX_NAME + "/_settings");
+                // update index settings should work
+                createTestIndex.setJsonEntity("{\"index.indexing.slowlog.level\": \"INFO\"}");
+                createTestIndex.setOptions(expectWarnings(EXPECTED_V8_WARNING));
+                client().performRequest(createTestIndex);
+
+                // ensure we were able to change the setting, despite it having no effect
+                Request indexSettingsRequest = new Request("GET", "/" + INDEX_NAME + "/_settings");
+                Map<String, Object> response = entityAsMap(client().performRequest(indexSettingsRequest));
+
+                var slowLogLevel = (String) (XContentMapValues.extractValue(
+                    INDEX_NAME + ".settings.index.indexing.slowlog.level",
+                    response
+                ));
+
+                // check that we can read our old index settings
+                assertThat(slowLogLevel, is("INFO"));
+            }
+            assertCount(INDEX_NAME, 2);
+        }
+    }
+
+    private void assertCount(String index, int countAtLeast) throws IOException {
+        Request searchTestIndexRequest = new Request("POST", "/" + index + "/_search");
+        searchTestIndexRequest.addParameter(TOTAL_HITS_AS_INT_PARAM, "true");
+        searchTestIndexRequest.addParameter("filter_path", "hits.total");
+        Response searchTestIndexResponse = client().performRequest(searchTestIndexRequest);
+        Map<String, Object> response = entityAsMap(searchTestIndexResponse);
+
+        var hitsTotal = (Integer) (XContentMapValues.extractValue("hits.total", response));
+
+        assertTrue(hitsTotal >= countAtLeast);
+    }
+
+    public static void updateIndexSettingsPermittingSlowlogDeprecationWarning(String index, Settings.Builder settings) throws IOException {
+        Request request = new Request("PUT", "/" + index + "/_settings");
+        request.setJsonEntity(org.elasticsearch.common.Strings.toString(settings.build()));
+        if (getOldClusterVersion().before(Version.V_7_17_9)) {
+            // There is a bug (fixed in 7.17.9 and 8.7.0 where deprecation warnings could leak into ClusterApplierService#applyChanges)
+            // Below warnings are set (and leaking) from an index in this test case
+            request.setOptions(expectVersionSpecificWarnings(v -> {
+                v.compatible(
+                    "[index.indexing.slowlog.level] setting was deprecated in Elasticsearch and will be removed in a future release! "
+                        + "See the breaking changes documentation for the next major version."
+                );
+            }));
+        }
+        client().performRequest(request);
+    }
+}

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/XPackIT.java → qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/XPackIT.java

@@ -7,6 +7,8 @@
  */
 package org.elasticsearch.upgrades;
 
+import com.carrotsearch.randomizedtesting.annotations.Name;
+
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.client.Request;
 import org.junit.Before;
@@ -20,7 +22,12 @@ import static org.junit.Assume.assumeThat;
  * Basic tests for simple xpack functionality that are only run if the
  * cluster is the on the default distribution.
  */
-public class XPackIT extends AbstractRollingTestCase {
+public class XPackIT extends ParameterizedRollingUpgradeTestCase {
+
+    public XPackIT(@Name("upgradedNodes") int upgradedNodes) {
+        super(upgradedNodes);
+    }
+
     @Before
     public void skipIfNotXPack() {
         assumeThat(
@@ -28,10 +35,9 @@ public class XPackIT extends AbstractRollingTestCase {
             System.getProperty("tests.distribution"),
             equalTo("default")
         );
-        assumeThat(
+        assumeTrue(
             "running this on the unupgraded cluster would change its state and it wouldn't work prior to 6.3 anyway",
-            CLUSTER_TYPE,
-            equalTo(ClusterType.UPGRADED)
+            isUpgradedCluster()
         );
         /*
          * *Mostly* we want this for when we're upgrading from pre-6.3's

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeWithOldIndexSettingsIT.java

@@ -1,131 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.upgrades;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.client.Request;
-import org.elasticsearch.client.Response;
-import org.elasticsearch.client.ResponseException;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.support.XContentMapValues;
-import org.elasticsearch.core.Strings;
-
-import java.io.IOException;
-import java.util.Map;
-
-import static org.elasticsearch.rest.action.search.RestSearchAction.TOTAL_HITS_AS_INT_PARAM;
-import static org.hamcrest.Matchers.is;
-
-public class UpgradeWithOldIndexSettingsIT extends AbstractRollingTestCase {
-
-    private static final String INDEX_NAME = "test_index_old_settings";
-    private static final String EXPECTED_WARNING = "[index.indexing.slowlog.level] setting was deprecated in Elasticsearch and will "
-        + "be removed in a future release! See the breaking changes documentation for the next major version.";
-
-    private static final String EXPECTED_V8_WARNING = "[index.indexing.slowlog.level] setting was deprecated in the previous Elasticsearch"
-        + " release and is removed in this release.";
-
-    @SuppressWarnings("unchecked")
-    public void testOldIndexSettings() throws Exception {
-        switch (CLUSTER_TYPE) {
-            case OLD -> {
-                Request createTestIndex = new Request("PUT", "/" + INDEX_NAME);
-                createTestIndex.setJsonEntity("{\"settings\": {\"index.indexing.slowlog.level\": \"WARN\"}}");
-                createTestIndex.setOptions(expectWarnings(EXPECTED_WARNING));
-                if (UPGRADE_FROM_VERSION.before(Version.V_8_0_0)) {
-                    // create index with settings no longer valid in 8.0
-                    client().performRequest(createTestIndex);
-                } else {
-                    assertTrue(
-                        expectThrows(ResponseException.class, () -> client().performRequest(createTestIndex)).getMessage()
-                            .contains("unknown setting [index.indexing.slowlog.level]")
-                    );
-
-                    Request createTestIndex1 = new Request("PUT", "/" + INDEX_NAME);
-                    client().performRequest(createTestIndex1);
-                }
-
-                // add some data
-                Request bulk = new Request("POST", "/_bulk");
-                bulk.addParameter("refresh", "true");
-                if (UPGRADE_FROM_VERSION.before(Version.V_8_0_0)) {
-                    bulk.setOptions(expectWarnings(EXPECTED_WARNING));
-                }
-                bulk.setJsonEntity(Strings.format("""
-                    {"index": {"_index": "%s"}}
-                    {"f1": "v1", "f2": "v2"}
-                    """, INDEX_NAME));
-                client().performRequest(bulk);
-            }
-            case MIXED -> {
-                // add some more data
-                Request bulk = new Request("POST", "/_bulk");
-                bulk.addParameter("refresh", "true");
-                if (UPGRADE_FROM_VERSION.before(Version.V_8_0_0)) {
-                    bulk.setOptions(expectWarnings(EXPECTED_WARNING));
-                }
-                bulk.setJsonEntity(Strings.format("""
-                    {"index": {"_index": "%s"}}
-                    {"f1": "v3", "f2": "v4"}
-                    """, INDEX_NAME));
-                client().performRequest(bulk);
-            }
-            case UPGRADED -> {
-                if (UPGRADE_FROM_VERSION.before(Version.V_8_0_0)) {
-                    Request createTestIndex = new Request("PUT", "/" + INDEX_NAME + "/_settings");
-                    // update index settings should work
-                    createTestIndex.setJsonEntity("{\"index.indexing.slowlog.level\": \"INFO\"}");
-                    createTestIndex.setOptions(expectWarnings(EXPECTED_V8_WARNING));
-                    client().performRequest(createTestIndex);
-
-                    // ensure we were able to change the setting, despite it having no effect
-                    Request indexSettingsRequest = new Request("GET", "/" + INDEX_NAME + "/_settings");
-                    Map<String, Object> response = entityAsMap(client().performRequest(indexSettingsRequest));
-
-                    var slowLogLevel = (String) (XContentMapValues.extractValue(
-                        INDEX_NAME + ".settings.index.indexing.slowlog.level",
-                        response
-                    ));
-
-                    // check that we can read our old index settings
-                    assertThat(slowLogLevel, is("INFO"));
-                }
-                assertCount(INDEX_NAME, 2);
-            }
-        }
-    }
-
-    private void assertCount(String index, int countAtLeast) throws IOException {
-        Request searchTestIndexRequest = new Request("POST", "/" + index + "/_search");
-        searchTestIndexRequest.addParameter(TOTAL_HITS_AS_INT_PARAM, "true");
-        searchTestIndexRequest.addParameter("filter_path", "hits.total");
-        Response searchTestIndexResponse = client().performRequest(searchTestIndexRequest);
-        Map<String, Object> response = entityAsMap(searchTestIndexResponse);
-
-        var hitsTotal = (Integer) (XContentMapValues.extractValue("hits.total", response));
-
-        assertTrue(hitsTotal >= countAtLeast);
-    }
-
-    public static void updateIndexSettingsPermittingSlowlogDeprecationWarning(String index, Settings.Builder settings) throws IOException {
-        Request request = new Request("PUT", "/" + index + "/_settings");
-        request.setJsonEntity(org.elasticsearch.common.Strings.toString(settings.build()));
-        if (UPGRADE_FROM_VERSION.before(Version.V_7_17_9)) {
-            // There is a bug (fixed in 7.17.9 and 8.7.0 where deprecation warnings could leak into ClusterApplierService#applyChanges)
-            // Below warnings are set (and leaking) from an index in this test case
-            request.setOptions(expectVersionSpecificWarnings(v -> {
-                v.compatible(
-                    "[index.indexing.slowlog.level] setting was deprecated in Elasticsearch and will be removed in a future release! "
-                        + "See the breaking changes documentation for the next major version."
-                );
-            }));
-        }
-        client().performRequest(request);
-    }
-}