Add DiscoveryNodeRole compatibility role for bwc tier serialization (#63581)

When introducing new roles, older versions of Elasticsearch aren't aware of these roles. This can
lead to a situation where an old (say 7.9.x) master node sees nodes in the cluster of version 7.10+
with "data_hot" or "data_content" roles, and thinks those roles are not eligible to hold data (since
the older node has no concept of these roles).

This adds a method to `DiscoveryNodeRole` where a role can return a compatibility role to be used
for serializing to an older Elasicsearch version. The new formalized data tier
roles (`data_content`, `data_hot`, `data_warm`, `data_cold`) uses this mechanism to serialize as
regular "data" roles when talking to an older Elasticsearch node. This will allow these nodes to
still contain data during a rolling upgrade where the master is upgraded last.

Relates to #60848

server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java

@@ -285,8 +285,9 @@ public class DiscoveryNode implements Writeable, ToXContentFragment {
         out.writeMap(attributes, StreamOutput::writeString, StreamOutput::writeString);
         out.writeVInt(roles.size());
         for (final DiscoveryNodeRole role : roles) {
-            out.writeString(role.roleName());
-            out.writeString(role.roleNameAbbreviation());
+            final DiscoveryNodeRole compatibleRole = role.getCompatibilityRole(out.getVersion());
+            out.writeString(compatibleRole.roleName());
+            out.writeString(compatibleRole.roleNameAbbreviation());
         }
         Version.writeVersion(version, out);
     }

server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeRole.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.cluster.node;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
@@ -81,6 +82,16 @@ public abstract class DiscoveryNodeRole implements Comparable<DiscoveryNodeRole>
         return false;
     }
 
+    /**
+     * When serializing a {@link DiscoveryNodeRole}, the role may not be available to nodes of
+     * previous versions, where the role had not yet been added. This method allows overriding
+     * the role that should be serialized when communicating to versions prior to the introduction
+     * of the discovery node role.
+     */
+    public DiscoveryNodeRole getCompatibilityRole(Version nodeVersion) {
+        return this;
+    }
+
     @Override
     public final boolean equals(Object o) {
         if (this == o) return true;

server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeTests.java

@@ -22,13 +22,16 @@ package org.elasticsearch.cluster.node;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.test.ESTestCase;
 
 import java.net.InetAddress;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
@@ -88,6 +91,53 @@ public class DiscoveryNodeTests extends ESTestCase {
         assertEquals(transportAddress.getPort(), serialized.getAddress().getPort());
     }
 
+    public void testDiscoveryNodeRoleWithOldVersion() throws Exception {
+        InetAddress inetAddress = InetAddress.getByAddress("name1", new byte[] { (byte) 192, (byte) 168, (byte) 0, (byte) 1});
+        TransportAddress transportAddress = new TransportAddress(inetAddress, randomIntBetween(0, 65535));
+
+        DiscoveryNodeRole customRole = new DiscoveryNodeRole("custom_role", "z") {
+            @Override
+            public Setting<Boolean> legacySetting() {
+                return null;
+            }
+
+            @Override
+            public DiscoveryNodeRole getCompatibilityRole(Version nodeVersion) {
+                if (nodeVersion.equals(Version.CURRENT)) {
+                    return this;
+                } else {
+                    return DiscoveryNodeRole.DATA_ROLE;
+                }
+            }
+        };
+
+        DiscoveryNode node = new DiscoveryNode("name1", "id1", transportAddress, emptyMap(),
+            Collections.singleton(customRole), Version.CURRENT);
+
+        {
+            BytesStreamOutput streamOutput = new BytesStreamOutput();
+            streamOutput.setVersion(Version.CURRENT);
+            node.writeTo(streamOutput);
+
+            StreamInput in = StreamInput.wrap(streamOutput.bytes().toBytesRef().bytes);
+            DiscoveryNode serialized = new DiscoveryNode(in);
+            assertThat(serialized.getRoles().stream().map(DiscoveryNodeRole::roleName).collect(Collectors.joining()),
+                equalTo("custom_role"));
+        }
+
+        {
+            BytesStreamOutput streamOutput = new BytesStreamOutput();
+            streamOutput.setVersion(Version.V_7_10_0);
+            node.writeTo(streamOutput);
+
+            StreamInput in = StreamInput.wrap(streamOutput.bytes().toBytesRef().bytes);
+            DiscoveryNode serialized = new DiscoveryNode(in);
+            assertThat(serialized.getRoles().stream().map(DiscoveryNodeRole::roleName).collect(Collectors.joining()),
+                equalTo("data"));
+        }
+
+    }
+
     public void testDiscoveryNodeIsRemoteClusterClientDefault() {
         runTestDiscoveryNodeIsRemoteClusterClient(Settings.EMPTY, true);
     }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTier.java

@@ -8,6 +8,7 @@ package org.elasticsearch.xpack.core;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
@@ -80,6 +81,11 @@ public class DataTier {
         public boolean canContainData() {
             return true;
         }
+
+        @Override
+        public DiscoveryNodeRole getCompatibilityRole(Version nodeVersion) {
+            return nodeVersion.before(Version.V_7_10_0) ? DiscoveryNodeRole.DATA_ROLE : this;
+        }
     };
 
     public static DiscoveryNodeRole DATA_HOT_NODE_ROLE = new DiscoveryNodeRole("data_hot", "h") {
@@ -97,6 +103,11 @@ public class DataTier {
         public boolean canContainData() {
             return true;
         }
+
+        @Override
+        public DiscoveryNodeRole getCompatibilityRole(Version nodeVersion) {
+            return nodeVersion.before(Version.V_7_10_0) ? DiscoveryNodeRole.DATA_ROLE : this;
+        }
     };
 
     public static DiscoveryNodeRole DATA_WARM_NODE_ROLE = new DiscoveryNodeRole("data_warm", "w") {
@@ -114,6 +125,11 @@ public class DataTier {
         public boolean canContainData() {
             return true;
         }
+
+        @Override
+        public DiscoveryNodeRole getCompatibilityRole(Version nodeVersion) {
+            return nodeVersion.before(Version.V_7_10_0) ? DiscoveryNodeRole.DATA_ROLE : this;
+        }
     };
 
     public static DiscoveryNodeRole DATA_COLD_NODE_ROLE = new DiscoveryNodeRole("data_cold", "c") {
@@ -131,6 +147,11 @@ public class DataTier {
         public boolean canContainData() {
             return true;
         }
+
+        @Override
+        public DiscoveryNodeRole getCompatibilityRole(Version nodeVersion) {
+            return nodeVersion.before(Version.V_7_10_0) ? DiscoveryNodeRole.DATA_ROLE : this;
+        }
     };
 
     public static boolean isContentNode(DiscoveryNode discoveryNode) {

x-pack/qa/mixed-tier-cluster/build.gradle

@@ -0,0 +1,61 @@
+apply plugin: 'elasticsearch.testclusters'
+apply plugin: 'elasticsearch.standalone-rest-test'
+apply from : "$rootDir/gradle/bwc-test.gradle"
+apply plugin: 'elasticsearch.rest-test'
+
+import org.elasticsearch.gradle.Version
+import org.elasticsearch.gradle.VersionProperties
+import org.elasticsearch.gradle.info.BuildParams
+import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
+
+dependencies {
+  testImplementation project(':x-pack:qa')
+}
+
+// Only run tests for 7.9+, since the node.roles setting was introduced in 7.9.0
+for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible.findAll { it.onOrAfter('7.9.0') }) {
+  if (bwcVersion == VersionProperties.getElasticsearchVersion()) {
+    // Not really a mixed cluster
+    continue;
+  }
+
+  String baseName = "v${bwcVersion}"
+
+  testClusters {
+    "${baseName}" {
+      versions = [bwcVersion.toString(), project.version]
+      numberOfNodes = 3
+      testDistribution = 'DEFAULT'
+      setting 'xpack.security.enabled', 'false'
+      setting 'xpack.watcher.enabled', 'false'
+      setting 'xpack.ml.enabled', 'false'
+      setting 'xpack.license.self_generated.type', 'trial'
+      nodes."${baseName}-0".setting 'node.roles', '["master"]'
+      // data_* roles were introduced in 7.10.0, so use 'data' for older versions
+      if (bwcVersion.before('7.10.0')) {
+        nodes."${baseName}-1".setting 'node.roles', '["data"]'
+      } else {
+        nodes."${baseName}-1".setting 'node.roles', '["data_content", "data_hot"]'
+      }
+      nodes."${baseName}-2".setting 'node.roles', '["master"]'
+    }
+  }
+
+  tasks.register("${baseName}#mixedClusterTest", StandaloneRestIntegTestTask) {
+    useCluster testClusters."${baseName}"
+    mustRunAfter(precommit)
+    doFirst {
+      // Getting the endpoints causes a wait for the cluster
+      println "Endpoints are: ${-> testClusters."${baseName}".allHttpSocketURI.join(",")}"
+      testClusters."${baseName}".nextNodeToNextVersion()
+
+      nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
+      nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
+    }
+    onlyIf { project.bwc_tests_enabled }
+  }
+
+  tasks.register(bwcTaskName(bwcVersion)) {
+    dependsOn "${baseName}#mixedClusterTest"
+  }
+}

x-pack/qa/mixed-tier-cluster/src/test/java/org/elasticsearch/mixed/DataTierMixedIT.java

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.mixed;
+
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.rest.ESRestTestCase;
+
+public class DataTierMixedIT extends ESRestTestCase {
+
+    public void testMixedTierCompatibility() throws Exception {
+        createIndex("test-index", Settings.builder()
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+            .build());
+        ensureGreen("test-index");
+    }
+}