Add "index" and "search" node roles with feature flag and setting (#90993)

This commit introduces the index and search node roles, 
along with a feature flag to enable them.

The feature flag is a convenient way to change the default 
roles enabled by Elasticsearch when no roles are provided, 
or when default roles are derived from empty settings, without 
having to change a lot of tests. The feature flag can only be 
enabled on snapshot builds for now.

The new roles are mutually exclusive with existing data ones. 
This verification is done in a separate component (for now, 
we may want this to be in server in the future) when both the 
feature flag and a specific setting is enabled, indicating that 
Elasticsearch runs in a specific mode where the default 
behavior can be changed.

docs/changelog/90993.yaml

@@ -0,0 +1,5 @@
+pr: 90993
+summary: Add "index" and "search" node roles with feature flag and setting
+area: Distributed
+type: feature
+issues: []

server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java

@@ -65,9 +65,11 @@ public class ClusterStatsIT extends ESIntegTestCase {
         expectedCounts.put(DiscoveryNodeRole.DATA_HOT_NODE_ROLE.roleName(), 1);
         expectedCounts.put(DiscoveryNodeRole.DATA_WARM_NODE_ROLE.roleName(), 1);
         expectedCounts.put(DiscoveryNodeRole.INGEST_ROLE.roleName(), 1);
+        expectedCounts.put(DiscoveryNodeRole.INDEX_ROLE.roleName(), 0);
         expectedCounts.put(DiscoveryNodeRole.MASTER_ROLE.roleName(), 1);
         expectedCounts.put(DiscoveryNodeRole.ML_ROLE.roleName(), 1);
         expectedCounts.put(DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE.roleName(), 1);
+        expectedCounts.put(DiscoveryNodeRole.SEARCH_ROLE.roleName(), 0);
         expectedCounts.put(DiscoveryNodeRole.TRANSFORM_ROLE.roleName(), 1);
         expectedCounts.put(DiscoveryNodeRole.VOTING_ONLY_NODE_ROLE.roleName(), 0);
         expectedCounts.put(ClusterStatsNodes.Counts.COORDINATING_ONLY, 0);

server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java

@@ -39,6 +39,29 @@ import static org.elasticsearch.node.NodeRoleSettings.NODE_ROLES_SETTING;
  */
 public class DiscoveryNode implements Writeable, ToXContentFragment {
 
+    /**
+     * Name of the setting used to enable stateless.
+     */
+    public static final String STATELESS_ENABLED_SETTING_NAME = "stateless.enabled";
+
+    /**
+     * Check if {@link #STATELESS_ENABLED_SETTING_NAME} is present and set to {@code true}, indicating that the node is
+     * part of a stateless deployment. When no settings are provided this method falls back to the value of the stateless feature flag;
+     * this is convenient for testing purpose as well as all behaviors that rely on node roles to be enabled/disabled by default when no
+     * settings are provided.
+     *
+     * @param settings the node settings
+     * @return true if {@link #STATELESS_ENABLED_SETTING_NAME} is present and set
+     */
+    public static boolean isStateless(final Settings settings) {
+        if (settings.isEmpty() == false) {
+            return settings.getAsBoolean(STATELESS_ENABLED_SETTING_NAME, false);
+        } else {
+            // Fallback on stateless feature flag when no settings are provided
+            return DiscoveryNodeRole.hasStatelessFeatureFlag();
+        }
+    }
+
     static final String COORDINATING_ONLY = "coordinating_only";
     public static final Version EXTERNAL_ID_VERSION = Version.V_8_3_0;
     public static final Comparator<DiscoveryNode> DISCOVERY_NODE_COMPARATOR = Comparator.comparing(DiscoveryNode::getName)

server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeRole.java

@@ -8,8 +8,10 @@
 
 package org.elasticsearch.cluster.node;
 
+import org.elasticsearch.Build;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.core.Booleans;
 
 import java.lang.reflect.Field;
 import java.util.Arrays;
@@ -26,6 +28,20 @@ import java.util.stream.Collectors;
  */
 public class DiscoveryNodeRole implements Comparable<DiscoveryNodeRole> {
 
+    /**
+     * A feature flag to indicate if stateless is available or not. This is useful to enable stateless specific behavior like node roles
+     * enabled by default. Defaults to false.
+     */
+    private static final String USE_STATELESS_SYSTEM_PROPERTY = "es.use_stateless";
+    private static final Boolean USE_STATELESS_FEATURE_FLAG;
+    static {
+        final Boolean useStateless = Booleans.parseBoolean(System.getProperty(USE_STATELESS_SYSTEM_PROPERTY), false);
+        if (useStateless && Build.CURRENT.isSnapshot() == false) {
+            throw new IllegalArgumentException("Enabling stateless usage is only supported in snapshot builds");
+        }
+        USE_STATELESS_FEATURE_FLAG = useStateless;
+    }
+
     private final String roleName;
 
     /**
@@ -141,7 +157,18 @@ public class DiscoveryNodeRole implements Comparable<DiscoveryNodeRole> {
     /**
      * Represents the role for a data node.
      */
-    public static final DiscoveryNodeRole DATA_ROLE = new DiscoveryNodeRole("data", "d", true);
+    public static final DiscoveryNodeRole DATA_ROLE = new DiscoveryNodeRole("data", "d", true) {
+
+        @Override
+        public boolean isEnabledByDefault(Settings settings) {
+            return DiscoveryNode.isStateless(settings) == false;
+        }
+
+        @Override
+        public void validateRoles(List<DiscoveryNodeRole> roles) {
+            ensureNoStatelessFeatureFlag(this);
+        }
+    };
 
     /**
      * Represents the role for a content node.
@@ -153,6 +180,10 @@ public class DiscoveryNodeRole implements Comparable<DiscoveryNodeRole> {
             return DiscoveryNode.hasRole(settings, DiscoveryNodeRole.DATA_ROLE);
         }
 
+        @Override
+        public void validateRoles(List<DiscoveryNodeRole> roles) {
+            ensureNoStatelessFeatureFlag(this);
+        }
     };
 
     /**
@@ -165,6 +196,10 @@ public class DiscoveryNodeRole implements Comparable<DiscoveryNodeRole> {
             return DiscoveryNode.hasRole(settings, DiscoveryNodeRole.DATA_ROLE);
         }
 
+        @Override
+        public void validateRoles(List<DiscoveryNodeRole> roles) {
+            ensureNoStatelessFeatureFlag(this);
+        }
     };
 
     /**
@@ -177,6 +212,10 @@ public class DiscoveryNodeRole implements Comparable<DiscoveryNodeRole> {
             return DiscoveryNode.hasRole(settings, DiscoveryNodeRole.DATA_ROLE);
         }
 
+        @Override
+        public void validateRoles(List<DiscoveryNodeRole> roles) {
+            ensureNoStatelessFeatureFlag(this);
+        }
     };
 
     /**
@@ -189,6 +228,10 @@ public class DiscoveryNodeRole implements Comparable<DiscoveryNodeRole> {
             return DiscoveryNode.hasRole(settings, DiscoveryNodeRole.DATA_ROLE);
         }
 
+        @Override
+        public void validateRoles(List<DiscoveryNodeRole> roles) {
+            ensureNoStatelessFeatureFlag(this);
+        }
     };
 
     /**
@@ -201,6 +244,10 @@ public class DiscoveryNodeRole implements Comparable<DiscoveryNodeRole> {
             return DiscoveryNode.hasRole(settings, DiscoveryNodeRole.DATA_ROLE);
         }
 
+        @Override
+        public void validateRoles(List<DiscoveryNodeRole> roles) {
+            ensureNoStatelessFeatureFlag(this);
+        }
     };
 
     /**
@@ -247,6 +294,27 @@ public class DiscoveryNodeRole implements Comparable<DiscoveryNodeRole> {
      */
     public static final DiscoveryNodeRole TRANSFORM_ROLE = new DiscoveryNodeRole("transform", "t");
 
+    /**
+     * Represents the role for an index node.
+     */
+    public static final DiscoveryNodeRole INDEX_ROLE = new DiscoveryNodeRole("index", "I", true) {
+
+        @Override
+        public boolean isEnabledByDefault(Settings settings) {
+            return DiscoveryNode.isStateless(settings);
+        }
+    };
+
+    /**
+     * Represents the role for a search node.
+     */
+    public static final DiscoveryNodeRole SEARCH_ROLE = new DiscoveryNodeRole("search", "S", true) {
+
+        public boolean isEnabledByDefault(Settings settings) {
+            return false;
+        }
+    };
+
     /**
      * Represents an unknown role. This can occur if a newer version adds a role that an older version does not know about, or a newer
      * version removes a role that an older version knows about.
@@ -335,4 +403,13 @@ public class DiscoveryNodeRole implements Comparable<DiscoveryNodeRole> {
         return maybeGetRoleFromRoleName(roleName).orElseThrow(() -> new IllegalArgumentException("unknown role [" + roleName + "]"));
     }
 
+    public static boolean hasStatelessFeatureFlag() {
+        return USE_STATELESS_FEATURE_FLAG;
+    }
+
+    private static void ensureNoStatelessFeatureFlag(DiscoveryNodeRole role) {
+        if (hasStatelessFeatureFlag()) {
+            throw new IllegalArgumentException("Role [" + role.roleName() + "] is only supported on non-stateless deployments");
+        }
+    }
 }

server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java

@@ -113,10 +113,12 @@ public class ClusterRerouteResponseTests extends ESTestCase {
                       "data_frozen",
                       "data_hot",
                       "data_warm",
+                      "index",
                       "ingest",
                       "master",
                       "ml",
                       "remote_cluster_client",
+                      "search",
                       "transform",
                       "voting_only"
                     ]

server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java

@@ -188,10 +188,12 @@ public class ClusterStateTests extends ESTestCase {
                     "data_frozen",
                     "data_hot",
                     "data_warm",
+                    "index",
                     "ingest",
                     "master",
                     "ml",
                     "remote_cluster_client",
+                    "search",
                     "transform",
                     "voting_only"
                   ]
@@ -406,10 +408,12 @@ public class ClusterStateTests extends ESTestCase {
                     "data_frozen",
                     "data_hot",
                     "data_warm",
+                    "index",
                     "ingest",
                     "master",
                     "ml",
                     "remote_cluster_client",
+                    "search",
                     "transform",
                     "voting_only"
                   ]
@@ -617,10 +621,12 @@ public class ClusterStateTests extends ESTestCase {
                     "data_frozen",
                     "data_hot",
                     "data_warm",
+                    "index",
                     "ingest",
                     "master",
                     "ml",
                     "remote_cluster_client",
+                    "search",
                     "transform",
                     "voting_only"
                   ]

server/src/test/java/org/elasticsearch/cluster/routing/allocation/DataTierTests.java

@@ -143,6 +143,9 @@ public class DataTierTests extends ESTestCase {
     private static List<DiscoveryNode> randomNodes(final int numNodes) {
         Set<DiscoveryNodeRole> allRoles = new HashSet<>(DiscoveryNodeRole.roles());
         allRoles.remove(DiscoveryNodeRole.DATA_ROLE);
+        // indexing and searching node role are mutually exclusive with data tiers roles
+        allRoles.remove(DiscoveryNodeRole.INDEX_ROLE);
+        allRoles.remove(DiscoveryNodeRole.SEARCH_ROLE);
         List<DiscoveryNode> nodesList = new ArrayList<>();
         for (int i = 0; i < numNodes; i++) {
             Map<String, String> attributes = new HashMap<>();

server/src/test/java/org/elasticsearch/health/node/DiskHealthIndicatorServiceTests.java

@@ -487,8 +487,8 @@ public class DiskHealthIndicatorServiceTests extends ESTestCase {
             result.symptom(),
             equalTo(
                 (numberOfRedNodes == 1 ? "1 node " : numberOfRedNodes + " nodes ")
-                    + "with roles: [data, data_cold, data_content, data_frozen, data_hot, data_warm, ingest, master, ml, "
-                    + "remote_cluster_client, transform, voting_only] "
+                    + "with roles: [data, data_cold, data_content, data_frozen, data_hot, data_warm, index, ingest, master, ml, "
+                    + "remote_cluster_client, search, transform, voting_only] "
                     + (numberOfRedNodes == 1 ? "is" : "are")
                     + " out of disk or running low on disk space."
             )

x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageIT.java

@@ -522,6 +522,8 @@ public class ReactiveStorageIT extends AutoscalingStorageIntegTestCase {
                     .stream()
                     .filter(DiscoveryNodeRole::canContainData)
                     .filter(r -> r != DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE)
+                    .filter(r -> r != DiscoveryNodeRole.SEARCH_ROLE)
+                    .filter(r -> r != DiscoveryNodeRole.INDEX_ROLE)
                     .sorted()
                     .collect(Collectors.toList())
             )

x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java

@@ -568,10 +568,12 @@ public class ClusterStatsMonitoringDocTests extends BaseMonitoringDocTestCase<Cl
                     "data_frozen": 0,
                     "data_hot": 0,
                     "data_warm": 0,
+                    "index": 0,
                     "ingest": 0,
                     "master": 1,
                     "ml": 0,
                     "remote_cluster_client": 0,
+                    "search": 0,
                     "transform": 0,
                     "voting_only": 0
                   },