Add Node Shutdown upgrade tests (#81506)

This PR adds full cluster restart and rolling upgrade tests,
to ensure that Node Shutdown handles BWC correctly.

Relates #70338

x-pack/plugin/shutdown/qa/full-cluster-restart/build.gradle

@@ -0,0 +1,98 @@
+import org.elasticsearch.gradle.internal.info.BuildParams
+import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
+
+apply plugin: 'elasticsearch.internal-testclusters'
+apply plugin: 'elasticsearch.standalone-rest-test'
+apply plugin: 'elasticsearch.bwc-test'
+
+dependencies {
+  // TODO: Remove core dependency and change tests to not use builders that are part of xpack-core.
+  // Currently needed for MlConfigIndexMappingsFullClusterRestartIT and SLM classes used in
+  // FullClusterRestartIT
+  testImplementation(testArtifact(project(xpackModule('core'))))
+  testImplementation(testArtifact(project(":qa:full-cluster-restart")))
+  testImplementation project(':x-pack:qa')
+}
+
+tasks.named("forbiddenPatterns") {
+  exclude '**/system_key'
+}
+
+String outputDir = "${buildDir}/generated-resources/${project.name}"
+
+tasks.register("copyTestNodeKeyMaterial", Copy) {
+  from project(':x-pack:plugin:core')
+    .files(
+      'src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem',
+      'src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt'
+    )
+  into outputDir
+}
+
+BuildParams.bwcVersions.withIndexCompatiple { bwcVersion, baseName ->
+    def baseCluster = testClusters.register(baseName) {
+        testDistribution = "DEFAULT"
+        versions = [bwcVersion.toString(), project.version]
+        numberOfNodes = 2
+        setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
+        user username: "test_user", password: "x-pack-test-password"
+
+        setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
+        // some tests rely on the translog not being flushed
+        setting 'indices.memory.shard_inactive_time', '60m'
+        setting 'xpack.security.enabled', 'true'
+        setting 'xpack.security.transport.ssl.enabled', 'true'
+        setting 'xpack.license.self_generated.type', 'trial'
+
+        extraConfigFile 'testnode.pem', file("${outputDir}/testnode.pem")
+        extraConfigFile 'testnode.crt', file("${outputDir}/testnode.crt")
+
+        keystore 'xpack.watcher.encryption_key', file("${project.projectDir}/src/test/resources/system_key")
+        setting 'xpack.watcher.encrypt_sensitive_data', 'true'
+
+        setting 'xpack.security.transport.ssl.key', 'testnode.pem'
+        setting 'xpack.security.transport.ssl.certificate', 'testnode.crt'
+        keystore 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode'
+
+        setting 'xpack.security.authc.api_key.enabled', 'true'
+        if (BuildParams.isSnapshotBuild() == false && bwcVersion.toString() == project.version) {
+            systemProperty 'es.index_mode_feature_flag_registered', 'true'
+        }
+    }
+
+    tasks.register("${baseName}#oldClusterTest", StandaloneRestIntegTestTask) {
+        mustRunAfter("precommit")
+        useCluster baseCluster
+        dependsOn "copyTestNodeKeyMaterial"
+        doFirst {
+            delete("${buildDir}/cluster/shared/repo/${baseName}")
+        }
+        systemProperty 'tests.is_old_cluster', 'true'
+    }
+
+    tasks.register("${baseName}#upgradedClusterTest", StandaloneRestIntegTestTask) {
+        mustRunAfter("precommit")
+        useCluster baseCluster
+        dependsOn "${baseName}#oldClusterTest"
+        doFirst {
+            if (BuildParams.isSnapshotBuild() == false) {
+                systemProperty 'es.index_mode_feature_flag_registered', 'true'
+            }
+            testClusters.named(baseName).get().goToNextVersion()
+        }
+        systemProperty 'tests.is_old_cluster', 'false'
+    }
+
+    String oldVersion = bwcVersion.toString().minus("-SNAPSHOT")
+    tasks.matching { it.name.startsWith("${baseName}#") && it.name.endsWith("ClusterTest") }.configureEach {
+        it.systemProperty 'tests.old_cluster_version', oldVersion
+        it.systemProperty 'tests.path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
+        it.nonInputProperties.systemProperty('tests.rest.cluster', baseCluster.map(c -> c.allHttpSocketURI.join(",")))
+        it.nonInputProperties.systemProperty('tests.clustername', baseName)
+    }
+
+    tasks.register(bwcTaskName(bwcVersion)) {
+        dependsOn "${baseName}#upgradedClusterTest"
+    }
+
+}

x-pack/plugin/shutdown/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java

@@ -0,0 +1,100 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.restart;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.upgrades.AbstractFullClusterRestartTestCase;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.json.JsonXContent;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.Matchers.anyOf;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.equalToIgnoringCase;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.notNullValue;
+
+public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
+
+    @Override
+    protected Settings restClientSettings() {
+        String token = "Basic " + Base64.getEncoder().encodeToString("test_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8));
+        return Settings.builder()
+            .put(ThreadContext.PREFIX + ".Authorization", token)
+            // we increase the timeout here to 90 seconds to handle long waits for a green
+            // cluster health. the waits for green need to be longer than a minute to
+            // account for delayed shards
+            .put(ESRestTestCase.CLIENT_SOCKET_TIMEOUT, "90s")
+            .build();
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testNodeShutdown() throws Exception {
+        assumeTrue("no shutdown in versions before " + Version.V_7_15_0, getOldClusterVersion().onOrAfter(Version.V_7_15_0));
+
+        if (isRunningAgainstOldCluster()) {
+            final Request getNodesReq = new Request("GET", "_nodes");
+            final Response getNodesResp = adminClient().performRequest(getNodesReq);
+            final Map<String, Object> nodes = (Map<String, Object>) entityAsMap(getNodesResp).get("nodes");
+            final String nodeIdToShutdown = randomFrom(nodes.keySet());
+
+            final Request putShutdownRequest = new Request("PUT", "_nodes/" + nodeIdToShutdown + "/shutdown");
+            try (XContentBuilder putBody = JsonXContent.contentBuilder()) {
+                putBody.startObject();
+                {
+                    // Use the types available from as early as possible
+                    final String type = randomFrom("restart", "remove");
+                    putBody.field("type", type);
+                    putBody.field("reason", this.getTestName());
+                }
+                putBody.endObject();
+                putShutdownRequest.setJsonEntity(Strings.toString(putBody));
+            }
+            assertOK(client().performRequest(putShutdownRequest));
+        }
+
+        assertBusy(() -> {
+            final Request getShutdownsReq = new Request("GET", "_nodes/shutdown");
+            final Response getShutdownsResp = client().performRequest(getShutdownsReq);
+            final Map<String, Object> stringObjectMap = entityAsMap(getShutdownsResp);
+
+            final List<Map<String, Object>> shutdowns = (List<Map<String, Object>>) stringObjectMap.get("nodes");
+            assertThat("there should be exactly one shutdown registered", shutdowns, hasSize(1));
+            final Map<String, Object> shutdown = shutdowns.get(0);
+            assertThat(shutdown.get("node_id"), notNullValue()); // Since we randomly determine the node ID, we can't check it
+            assertThat(shutdown.get("reason"), equalTo(this.getTestName()));
+            assertThat(
+                (String) shutdown.get("status"),
+                anyOf(
+                    Arrays.stream(SingleNodeShutdownMetadata.Status.values())
+                        .map(value -> equalToIgnoringCase(value.toString()))
+                        .collect(Collectors.toList())
+                )
+            );
+        }, 30, TimeUnit.SECONDS);
+    }
+
+    @Override
+    protected void deleteAllNodeShutdownMetadata() throws IOException {
+        // do not delete node shutdown
+    }
+}

x-pack/plugin/shutdown/qa/full-cluster-restart/src/test/resources/system_key

@@ -0,0 +1 @@
+ь{Ю▌█Гю+°dTI;f└┌█⌠л╜╖l▀╥²|╞}┼jВЩУDЬvYWЪV5и┤K╢h╘8┼▀н╙P·
z~╡╫у┐a▒),$jд│.И╦шГж^▌Хw╔и╢х░38╫v ├}╞▀|╘^[ УFСь╢√█"т▒⌡г≈√╢

x-pack/plugin/shutdown/qa/rolling-upgrade/build.gradle

@@ -0,0 +1,166 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import org.elasticsearch.gradle.VersionProperties
+import org.elasticsearch.gradle.internal.info.BuildParams
+import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
+
+apply plugin: 'elasticsearch.internal-testclusters'
+apply plugin: 'elasticsearch.standalone-rest-test'
+apply plugin: 'elasticsearch.bwc-test'
+apply plugin: 'elasticsearch.rest-resources'
+
+dependencies {
+  testImplementation project(':x-pack:qa')
+  testImplementation project(':client:rest-high-level')
+}
+
+restResources {
+  restApi {
+    include '*'
+  }
+}
+
+tasks.named("forbiddenPatterns").configure {
+  exclude '**/system_key'
+}
+
+String outputDir = "${buildDir}/generated-resources/${project.name}"
+
+tasks.register("copyTestNodeKeyMaterial", Copy) {
+  from project(':x-pack:plugin:core').files('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem',
+    'src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt')
+  into outputDir
+}
+
+BuildParams.bwcVersions.withWireCompatiple { bwcVersion, baseName ->
+  String oldVersion = bwcVersion.toString()
+
+  // SearchableSnapshotsRollingUpgradeIT uses a specific repository to not interfere with other tests
+  String searchableSnapshotRepository = "${buildDir}/cluster/shared/searchable-snapshots-repo/${baseName}"
+
+  def baseCluster = testClusters.register(baseName) {
+    testDistribution = "DEFAULT"
+    versions = [oldVersion, project.version]
+    numberOfNodes = 3
+
+    setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
+    setting 'path.repo', "['${buildDir}/cluster/shared/repo/${baseName}', '${searchableSnapshotRepository}']"
+    setting 'xpack.license.self_generated.type', 'trial'
+    setting 'xpack.security.enabled', 'true'
+    setting 'xpack.security.transport.ssl.enabled', 'true'
+    setting 'xpack.security.authc.token.enabled', 'true'
+    setting 'xpack.security.authc.token.timeout', '60m'
+    setting 'xpack.security.authc.api_key.enabled', 'true'
+    setting 'xpack.security.audit.enabled', 'true'
+    setting 'xpack.security.transport.ssl.key', 'testnode.pem'
+    setting 'xpack.security.transport.ssl.certificate', 'testnode.crt'
+    keystore 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode'
+
+    if (bwcVersion.onOrAfter('7.0.0')) {
+      setting 'xpack.security.authc.realms.file.file1.order', '0'
+      setting 'xpack.security.authc.realms.native.native1.order', '1'
+    } else {
+      setting 'xpack.security.authc.realms.file1.type', 'file'
+      setting 'xpack.security.authc.realms.file1.order', '0'
+      setting 'xpack.security.authc.realms.native1.type', 'native'
+      setting 'xpack.security.authc.realms.native1.order', '1'
+    }
+    if (bwcVersion.onOrAfter('6.6.0')) {
+      setting 'ccr.auto_follow.wait_for_metadata_timeout', '1s'
+    }
+    if (bwcVersion.onOrAfter('7.11.0')) {
+      extraConfigFile 'operator_users.yml', file("${project.projectDir}/src/test/resources/operator_users.yml")
+      setting 'xpack.security.operator_privileges.enabled', "true"
+      user username: "non_operator", password: 'x-pack-test-password', role: "superuser"
+    }
+
+    user username: "test_user", password: "x-pack-test-password"
+
+    extraConfigFile 'testnode.pem', file("$outputDir/testnode.pem")
+    extraConfigFile 'testnode.crt', file("$outputDir/testnode.crt")
+
+    keystore 'xpack.watcher.encryption_key', file("${project.projectDir}/src/test/resources/system_key")
+    setting 'xpack.watcher.encrypt_sensitive_data', 'true'
+
+    // Old versions of the code contain an invalid assertion that trips
+    // during tests.  Versions 5.6.9 and 6.2.4 have been fixed by removing
+    // the assertion, but this is impossible for released versions.
+    // However, released versions run without assertions, so end users won't
+    // be suffering the effects.  This argument effectively removes the
+    // incorrect assertion from the older versions used in the BWC tests.
+    if (bwcVersion.before('5.6.9') || (bwcVersion.onOrAfter('6.0.0') && bwcVersion.before('6.2.4'))) {
+      jvmArgs '-da:org.elasticsearch.xpack.monitoring.exporter.http.HttpExportBulk'
+    }
+    setting 'logger.org.elasticsearch.xpack.watcher', 'DEBUG'
+
+    if (bwcVersion.onOrAfter('7.12.0')) {
+      setting 'xpack.searchable.snapshot.shared_cache.size', '10mb'
+    }
+  }
+
+  tasks.register("${baseName}#oldClusterTest", StandaloneRestIntegTestTask) {
+    useCluster baseCluster
+    mustRunAfter("precommit")
+    dependsOn "copyTestNodeKeyMaterial"
+    doFirst {
+      delete("${buildDir}/cluster/shared/repo/${baseName}")
+      delete("${searchableSnapshotRepository}")
+    }
+
+    systemProperty 'tests.rest.suite', 'old_cluster'
+    systemProperty 'tests.upgrade_from_version', oldVersion
+    systemProperty 'tests.path.searchable.snapshots.repo', searchableSnapshotRepository
+    nonInputProperties.systemProperty('tests.rest.cluster', baseCluster.map(c->c.allHttpSocketURI.join(",")))
+    nonInputProperties.systemProperty('tests.clustername', baseName)
+  }
+
+  tasks.register("${baseName}#oneThirdUpgradedTest", StandaloneRestIntegTestTask) {
+    dependsOn "${baseName}#oldClusterTest"
+    useCluster baseCluster
+    doFirst {
+      baseCluster.get().nextNodeToNextVersion()
+    }
+    nonInputProperties.systemProperty('tests.rest.cluster', baseCluster.map(c->c.allHttpSocketURI.join(",")))
+    nonInputProperties.systemProperty('tests.clustername', baseName)
+    systemProperty 'tests.rest.suite', 'mixed_cluster'
+    systemProperty 'tests.first_round', 'true'
+    systemProperty 'tests.upgrade_from_version', oldVersion
+    systemProperty 'tests.path.searchable.snapshots.repo', searchableSnapshotRepository
+  }
+
+  tasks.register("${baseName}#twoThirdsUpgradedTest", StandaloneRestIntegTestTask) {
+    dependsOn "${baseName}#oneThirdUpgradedTest"
+    useCluster baseCluster
+    doFirst {
+      baseCluster.get().nextNodeToNextVersion()
+    }
+    nonInputProperties.systemProperty('tests.rest.cluster', "${-> baseCluster.get().allHttpSocketURI.join(",")}")
+    nonInputProperties.systemProperty('tests.clustername', baseName)
+    systemProperty 'tests.rest.suite', 'mixed_cluster'
+    systemProperty 'tests.first_round', 'false'
+    systemProperty 'tests.upgrade_from_version', oldVersion
+    systemProperty 'tests.path.searchable.snapshots.repo', searchableSnapshotRepository
+  }
+
+  tasks.register("${baseName}#upgradedClusterTest", StandaloneRestIntegTestTask) {
+    dependsOn "${baseName}#twoThirdsUpgradedTest"
+    useCluster baseCluster
+    doFirst {
+      baseCluster.get().nextNodeToNextVersion()
+    }
+    nonInputProperties.systemProperty('tests.rest.cluster', "${-> baseCluster.get().allHttpSocketURI.join(",")}")
+    nonInputProperties.systemProperty('tests.clustername', baseName)
+    systemProperty 'tests.rest.suite', 'upgraded_cluster'
+    systemProperty 'tests.upgrade_from_version', oldVersion
+    systemProperty 'tests.path.searchable.snapshots.repo', searchableSnapshotRepository
+  }
+
+  tasks.register(bwcTaskName(bwcVersion)) {
+    dependsOn "${baseName}#upgradedClusterTest"
+  }
+}

x-pack/plugin/shutdown/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java

@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.upgrades;
+
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.xpack.test.SecuritySettingsSourceField;
+
+import java.io.IOException;
+
+public abstract class AbstractUpgradeTestCase extends ESRestTestCase {
+
+    private static final String BASIC_AUTH_VALUE = basicAuthHeaderValue(
+        "test_user",
+        new SecureString(SecuritySettingsSourceField.TEST_PASSWORD)
+    );
+
+    enum ClusterType {
+        OLD,
+        MIXED,
+        UPGRADED;
+
+        public static ClusterType parse(String value) {
+            switch (value) {
+                case "old_cluster":
+                    return OLD;
+                case "mixed_cluster":
+                    return MIXED;
+                case "upgraded_cluster":
+                    return UPGRADED;
+                default:
+                    throw new AssertionError("unknown cluster type: " + value);
+            }
+        }
+    }
+
+    protected static final ClusterType CLUSTER_TYPE = ClusterType.parse(System.getProperty("tests.rest.suite"));
+    protected static final boolean FIRST_MIXED_ROUND = Boolean.parseBoolean(System.getProperty("tests.first_round", "false"));
+
+    @Override
+    protected Settings restClientSettings() {
+        return Settings.builder()
+            .put(ThreadContext.PREFIX + ".Authorization", BASIC_AUTH_VALUE)
+
+            // increase the timeout here to 90 seconds to handle long waits for a green
+            // cluster health. the waits for green need to be longer than a minute to
+            // account for delayed shards
+            .put(ESRestTestCase.CLIENT_SOCKET_TIMEOUT, "90s")
+
+            .build();
+    }
+
+    @Override
+    protected void deleteAllNodeShutdownMetadata() throws IOException {
+        // do not delete node shutdown
+    }
+}

x-pack/plugin/shutdown/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/NodeShutdownUpgradeIT.java

@@ -0,0 +1,133 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.upgrades;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.json.JsonXContent;
+import org.hamcrest.Matcher;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.Matchers.allOf;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.hasEntry;
+
+public class NodeShutdownUpgradeIT extends AbstractUpgradeTestCase {
+    List<String> namesSorted;
+    Map<String, String> nodeNameToIdMap;
+
+    @SuppressWarnings("unchecked")
+    @Before
+    public void init() throws IOException {
+        final Request getNodesReq = new Request("GET", "_nodes");
+        final Response getNodesResp = adminClient().performRequest(getNodesReq);
+        final Map<String, Map<String, Object>> nodes = (Map<String, Map<String, Object>>) entityAsMap(getNodesResp).get("nodes");
+        nodeNameToIdMap = nodes.entrySet().stream().collect(Collectors.toMap(e -> (String) (e.getValue().get("name")), e -> e.getKey()));
+        namesSorted = nodeNameToIdMap.keySet().stream().sorted().collect(Collectors.toList());
+    }
+
+    public void testShutdown() throws Exception {
+        String nodeIdToShutdown;
+        switch (CLUSTER_TYPE) {
+            case OLD:
+                nodeIdToShutdown = nodeIdToShutdown(0);
+                assertOK(client().performRequest(shutdownNode(nodeIdToShutdown)));
+
+                assertBusy(() -> assertThat(getShutdownStatus(), containsInAnyOrder(shutdownStatusCompleteFor(0))));
+                break;
+
+            case MIXED:
+                if (FIRST_MIXED_ROUND) {
+                    // after upgrade the record still exist
+                    assertBusy(() -> assertThat(getShutdownStatus(), containsInAnyOrder(shutdownStatusCompleteFor(0))));
+
+                    nodeIdToShutdown = nodeIdToShutdown(1);
+                    assertOK(client().performRequest(shutdownNode(nodeIdToShutdown)));
+
+                    assertBusy(
+                        () -> assertThat(
+                            getShutdownStatus(),
+                            containsInAnyOrder(shutdownStatusCompleteFor(0), shutdownStatusCompleteFor(1))
+                        )
+                    );
+
+                } else {
+                    assertBusy(
+                        () -> assertThat(
+                            getShutdownStatus(),
+                            containsInAnyOrder(shutdownStatusCompleteFor(0), shutdownStatusCompleteFor(1))
+                        )
+                    );
+
+                    nodeIdToShutdown = nodeIdToShutdown(2);
+                    assertOK(client().performRequest(shutdownNode(nodeIdToShutdown)));
+
+                    assertBusy(
+                        () -> assertThat(
+                            getShutdownStatus(),
+                            containsInAnyOrder(shutdownStatusCompleteFor(0), shutdownStatusCompleteFor(1), shutdownStatusCompleteFor(2))
+                        )
+                    );
+                }
+
+                break;
+
+            case UPGRADED:
+                assertBusy(
+                    () -> assertThat(
+                        getShutdownStatus(),
+                        containsInAnyOrder(shutdownStatusCompleteFor(0), shutdownStatusCompleteFor(1), shutdownStatusCompleteFor(2))
+                    )
+                );
+                break;
+            default:
+                throw new UnsupportedOperationException("Unknown cluster type [" + CLUSTER_TYPE + "]");
+        }
+    }
+
+    private Matcher<Map<String, Object>> shutdownStatusCompleteFor(int i) {
+        return allOf(
+            hasEntry("node_id", nodeIdToShutdown(i)),
+            hasEntry("reason", this.getTestName()),
+            hasEntry("status", SingleNodeShutdownMetadata.Status.COMPLETE.toString())
+        );
+    }
+
+    @SuppressWarnings("unchecked")
+    private List<Map<String, Object>> getShutdownStatus() throws IOException {
+        final Request getShutdownsReq = new Request("GET", "_nodes/shutdown");
+        final Response getShutdownsResp = client().performRequest(getShutdownsReq);
+        return (List<Map<String, Object>>) entityAsMap(getShutdownsResp).get("nodes");
+    }
+
+    private Request shutdownNode(String nodeIdToShutdown) throws IOException {
+        final Request putShutdownRequest = new Request("PUT", "_nodes/" + nodeIdToShutdown + "/shutdown");
+        try (XContentBuilder putBody = JsonXContent.contentBuilder()) {
+            putBody.startObject();
+            {
+                putBody.field("type", "restart");
+                putBody.field("reason", this.getTestName());
+            }
+            putBody.endObject();
+            putShutdownRequest.setJsonEntity(Strings.toString(putBody));
+        }
+        return putShutdownRequest;
+    }
+
+    private String nodeIdToShutdown(int nodeNumber) {
+        final String nodeName = namesSorted.get(nodeNumber);
+        return nodeNameToIdMap.get(nodeName);
+    }
+}

x-pack/plugin/shutdown/qa/rolling-upgrade/src/test/resources/operator_users.yml

@@ -0,0 +1,2 @@
+operator:
+  - usernames: ["test_user"]

x-pack/plugin/shutdown/qa/rolling-upgrade/src/test/resources/system_key

@@ -0,0 +1 @@
+ь{Ю▌█Гю+°dTI;f└┌█⌠л╜╖l▀╥²|╞}┼jВЩУDЬvYWЪV5и┤K╢h╘8┼▀н╙P·
z~╡╫у┐a▒),$jд│.И╦шГж^▌Хw╔и╢х░38╫v ├}╞▀|╘^[ УFСь╢√█"т▒⌡г≈√╢