Add migrate to data tiers API (#74264)

This adds the _ilm/migrate_to_data_tiers API to expose the service for
migrating the elasticsearch abstractions (indices, ILM policies and an 
optional legacy template to delete) to data tiers routing allocation 
(away from custom node attributes)

docs/reference/ilm/apis/ilm-api.asciidoc

@@ -1,7 +1,7 @@
 [[index-lifecycle-management-api]]
 == {ilm-cap} APIs
 
-You use the following APIs to set up policies to automatically manage the index lifecycle. 
+You use the following APIs to set up policies to automatically manage the index lifecycle.
 For more information about {ilm} ({ilm-init}), see <<index-lifecycle-management>>.
 
 [discrete]
@@ -28,6 +28,7 @@ For more information about {ilm} ({ilm-init}), see <<index-lifecycle-management>
 * <<ilm-start,Start {ilm-init}>>
 * <<ilm-stop,Stop {ilm-init}>>
 * <<ilm-explain-lifecycle,Explain API>>
+* <<ilm-migrate-to-data-tiers,Migrate to data tiers routing API>>
 
 
 include::put-lifecycle.asciidoc[]
@@ -42,3 +43,4 @@ include::get-status.asciidoc[]
 include::explain.asciidoc[]
 include::start.asciidoc[]
 include::stop.asciidoc[]
+include::migrate-to-data-tiers.asciidoc[]

docs/reference/ilm/apis/migrate-to-data-tiers.asciidoc

@@ -0,0 +1,130 @@
+[role="xpack"]
+[testenv="basic"]
+[[ilm-migrate-to-data-tiers]]
+=== Migrate to data tiers routing API
+++++
+<titleabbrev>Migrate indices and ILM policies to data tiers routing</titleabbrev>
+++++
+
+Switches the indices and ILM policies from using custom node attributes and
+<<shard-allocation-filtering, attribute-based allocation filters>> to using <<data-tiers, data tiers>>, and
+optionally deletes one legacy index template.
+Using node roles enables {ilm-init} to <<data-tier-migration, automatically move the indices>> between
+data tiers.
+
+Migrating away from custom node attributes routing can be manually performed
+as indicated in the <<migrate-index-allocation-filters, Migrate index allocation
+filters to node roles>> page.
+
+This API provides an automated way of executing three out of the four manual steps listed
+in the <<data-tier-migration, migration guide>>:
+
+. <<stop-setting-custom-hot-attribute, Stop setting the custom hot attribute on new indices>>
+. <<remove-custom-allocation-settings, Remove custom allocation settings from existing {ilm-init} policies>>
+. <<set-tier-preference, Replace custom allocation settings from existing indices>> with the corresponding <<data-tier-shard-filtering,tier preference>>
+
+[[ilm-migrate-to-data-tiers-request]]
+==== {api-request-title}
+
+`POST /_ilm/migrate_to_data_tiers`
+
+The API accepts an optional body that allows you to specify:
+
+- The legacy index template name to delete. Defaults to none.
+- The name of the custom node attribute used for the indices and ILM policies allocation filtering.
+Defaults to `data`.
+
+[[ilm-migrate-to-data-tiers-prereqs]]
+==== {api-prereq-title}
+
+* {ilm-init} must be stopped before performing the migration. Use the <<ilm-stop-request, stop ILM API>>
+to stop {ilm-init} and <<ilm-get-status-request, get status API>> to wait until the
+reported operation mode is `STOPPED`.
+
+[[ilm-migrate-to-data-tiers-example]]
+==== {api-examples-title}
+
+The following example migrates the indices and ILM policies away from defining
+custom allocation filtering using the `custom_attribute_name` node attribute, and
+deletes legacy template with name `global-template` if it exists in the system.
+
+////
+[source,console]
+----
+POST _ilm/stop
+
+PUT _template/global-template
+{
+  "index_patterns": ["migrate-to-tiers-*"],
+  "settings": {
+     "index.routing.allocation.require.custom_attribute_name": "hot"
+  }
+}
+
+PUT warm-index-to-migrate-000001
+{
+  "settings": {
+    "index.routing.allocation.require.custom_attribute_name": "warm"
+  }
+}
+
+PUT _ilm/policy/policy_with_allocate_action
+{
+  "policy": {
+    "phases": {
+      "warm": {
+        "actions": {
+          "allocate": {
+            "require": {
+              "custom_attribute_name": "warm"
+            }
+          }
+        }
+      },
+      "delete": {
+        "min_age": "30d",
+        "actions": {
+          "delete": {}
+        }
+      }
+    }
+  }
+}
+----
+// TESTSETUP
+
+[source,console]
+----
+DELETE warm-index-to-migrate-000001
+
+DELETE _ilm/policy/policy_with_allocate_action
+
+POST _ilm/start
+----
+// TEARDOWN
+////
+
+[source,console]
+----------------------------------------------------------------
+POST /_ilm/migrate_to_data_tiers
+{
+  "legacy_template_to_delete": "global-template",
+  "node_attribute": "custom_attribute_name"
+}
+----------------------------------------------------------------
+
+If the request succeeds, a response like the following will be received:
+
+[source,console-result]
+------------------------------------------------------------------------------
+{
+  "removed_legacy_template":"global-template", <1>
+  "migrated_ilm_policies":["policy_with_allocate_action"], <2>
+  "migrated_indices":["warm-index-to-migrate-000001"] <3>
+}
+------------------------------------------------------------------------------
+
+<1> Shows the name of the legacy index template that was deleted. This will be missing
+if no legacy index template was deleted.
+<2> The ILM policies that were updated.
+<3> The indices that were migrated to <<data-tier-shard-filtering,tier preference>> routing.

rest-api-spec/src/main/resources/rest-api-spec/api/ilm.migrate_to_data_tiers.json

@@ -0,0 +1,29 @@
+{
+  "ilm.migrate_to_data_tiers":{
+    "documentation":{
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-migrate-to-data-tiers.html",
+      "description": "Migrates the indices and ILM policies away from custom node attribute allocation routing to data tiers routing"
+    },
+    "stability":"stable",
+    "visibility":"public",
+    "headers":{
+      "accept": [ "application/json"],
+      "content_type": ["application/json"]
+    },
+    "url":{
+      "paths":[
+        {
+          "path":"/_ilm/migrate_to_data_tiers",
+          "methods":[
+            "POST"
+          ]
+        }
+      ]
+    },
+    "params":{},
+    "body":{
+      "description":"Optionally specify a legacy index template name to delete and optionally specify a node attribute name used for index shard routing (defaults to \"data\")",
+      "required":false
+    }
+  }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/action/MigrateToDataTiersAction.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.xpack.cluster.action;
+
+import org.elasticsearch.action.ActionType;
+
+public class MigrateToDataTiersAction extends ActionType<MigrateToDataTiersResponse> {
+
+    public static final MigrateToDataTiersAction INSTANCE = new MigrateToDataTiersAction();
+    public static final String NAME = "cluster:admin/migrate_to_data_tiers";
+
+    private MigrateToDataTiersAction() {
+        super(NAME, MigrateToDataTiersResponse::new);
+    }
+
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/action/MigrateToDataTiersRequest.java

@@ -0,0 +1,113 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.cluster.action;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.support.master.AcknowledgedRequest;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ParseField;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.core.Nullable;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class MigrateToDataTiersRequest extends AcknowledgedRequest<MigrateToDataTiersRequest> {
+
+    private static final ParseField LEGACY_TEMPLATE_TO_DELETE = new ParseField("legacy_template_to_delete");
+    private static final ParseField NODE_ATTRIBUTE_NAME = new ParseField("node_attribute");
+
+    @SuppressWarnings("unchecked")
+    public static final ConstructingObjectParser<MigrateToDataTiersRequest, Void> PARSER = new ConstructingObjectParser<>("index_template",
+        false,
+        a -> new MigrateToDataTiersRequest((String) a[0], (String) a[1]));
+
+    static {
+        PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), LEGACY_TEMPLATE_TO_DELETE);
+        PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), NODE_ATTRIBUTE_NAME);
+    }
+
+    /**
+     * Represents the name of node attribute used for index shard allocation filtering (usually `data`)
+     */
+    @Nullable
+    private final String nodeAttributeName;
+
+    /**
+     * Represents the name of the legacy (v1) index template to delete.
+     */
+    @Nullable
+    private final String legacyTemplateToDelete;
+
+    public static MigrateToDataTiersRequest parse(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
+
+    public MigrateToDataTiersRequest(@Nullable String legacyTemplateToDelete, @Nullable String nodeAttributeName) {
+        this.legacyTemplateToDelete = legacyTemplateToDelete;
+        this.nodeAttributeName = nodeAttributeName;
+    }
+
+    public MigrateToDataTiersRequest() {
+        this(null, null);
+    }
+
+    public MigrateToDataTiersRequest(StreamInput in) throws IOException {
+        super(in);
+        legacyTemplateToDelete = in.readOptionalString();
+        nodeAttributeName = in.readOptionalString();
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        return null;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeOptionalString(legacyTemplateToDelete);
+        out.writeOptionalString(nodeAttributeName);
+    }
+
+    public String getNodeAttributeName() {
+        return nodeAttributeName;
+    }
+
+    public String getLegacyTemplateToDelete() {
+        return legacyTemplateToDelete;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        MigrateToDataTiersRequest that = (MigrateToDataTiersRequest) o;
+        return Objects.equals(nodeAttributeName, that.nodeAttributeName) && Objects.equals(legacyTemplateToDelete,
+            that.legacyTemplateToDelete);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(nodeAttributeName, legacyTemplateToDelete);
+    }
+
+    @Override
+    public String toString() {
+        return "MigrateToDataTiersRequest{" +
+            "nodeAttributeName='" + nodeAttributeName + '\'' +
+            ", legacyTemplateToDelete='" + legacyTemplateToDelete + '\'' +
+            '}';
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/action/MigrateToDataTiersResponse.java

@@ -0,0 +1,76 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.cluster.action;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ParseField;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.core.Nullable;
+
+import java.io.IOException;
+import java.util.List;
+
+public class MigrateToDataTiersResponse extends ActionResponse implements ToXContentObject {
+
+    public static final ParseField REMOVED_LEGACY_TEMPLATE = new ParseField("removed_legacy_template");
+    public static final ParseField MIGRATED_INDICES = new ParseField("migrated_indices");
+    public static final ParseField MIGRATED_ILM_POLICIES = new ParseField("migrated_ilm_policies");
+
+    @Nullable
+    private final String removedIndexTemplateName;
+    private final List<String> migratedPolicies;
+    private final List<String> migratedIndices;
+
+    public MigrateToDataTiersResponse(@Nullable String removedIndexTemplateName, List<String> migratedPolicies,
+                                      List<String> migratedIndices) {
+        this.removedIndexTemplateName = removedIndexTemplateName;
+        this.migratedPolicies = migratedPolicies;
+        this.migratedIndices = migratedIndices;
+    }
+
+    public MigrateToDataTiersResponse(StreamInput in) throws IOException {
+        super(in);
+        removedIndexTemplateName = in.readOptionalString();
+        migratedPolicies = in.readStringList();
+        migratedIndices = in.readStringList();
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        if (this.removedIndexTemplateName != null) {
+            builder.field(REMOVED_LEGACY_TEMPLATE.getPreferredName(), this.removedIndexTemplateName);
+        }
+        if (migratedPolicies.size() > 0) {
+            builder.startArray(MIGRATED_ILM_POLICIES.getPreferredName());
+            for (String policy : migratedPolicies) {
+                builder.value(policy);
+            }
+            builder.endArray();
+        }
+        if (migratedIndices.size() > 0) {
+            builder.startArray(MIGRATED_INDICES.getPreferredName());
+            for (String index : migratedIndices) {
+                builder.value(index);
+            }
+            builder.endArray();
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeOptionalString(removedIndexTemplateName);
+        out.writeStringCollection(migratedPolicies);
+        out.writeStringCollection(migratedIndices);
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java

@@ -33,6 +33,7 @@ import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.rollup.RollupV2;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.xpack.ccr.CCRInfoTransportAction;
+import org.elasticsearch.xpack.cluster.action.MigrateToDataTiersAction;
 import org.elasticsearch.xpack.core.action.XPackInfoAction;
 import org.elasticsearch.xpack.core.action.XPackUsageAction;
 import org.elasticsearch.xpack.core.aggregatemetric.AggregateMetricFeatureSetUsage;
@@ -396,6 +397,8 @@ public class XPackClientPlugin extends Plugin implements ActionPlugin, NetworkPl
             DeleteSnapshotLifecycleAction.INSTANCE,
             ExecuteSnapshotLifecycleAction.INSTANCE,
             GetSnapshotLifecycleStatsAction.INSTANCE,
+            MigrateToDataTiersAction.INSTANCE,
+
             // Freeze
             FreezeIndexAction.INSTANCE,
             // Data Frame

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/action/MigrateToDataTiersRequestTests.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.cluster.action;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+
+import java.io.IOException;
+
+public class MigrateToDataTiersRequestTests extends AbstractWireSerializingTestCase<MigrateToDataTiersRequest> {
+
+    @Override
+    protected Writeable.Reader<MigrateToDataTiersRequest> instanceReader() {
+        return MigrateToDataTiersRequest::new;
+    }
+
+    @Override
+    protected MigrateToDataTiersRequest createTestInstance() {
+        return new MigrateToDataTiersRequest(randomAlphaOfLength(10), randomAlphaOfLength(10));
+    }
+
+    @Override
+    protected MigrateToDataTiersRequest mutateInstance(MigrateToDataTiersRequest instance) throws IOException {
+        return randomValueOtherThan(instance, this::createTestInstance);
+    }
+}

x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java

@@ -0,0 +1,244 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack;
+
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.util.EntityUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.xpack.cluster.action.MigrateToDataTiersResponse;
+import org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDecider;
+import org.elasticsearch.xpack.core.ilm.AllocateAction;
+import org.elasticsearch.xpack.core.ilm.DeleteAction;
+import org.elasticsearch.xpack.core.ilm.ForceMergeAction;
+import org.elasticsearch.xpack.core.ilm.LifecycleAction;
+import org.elasticsearch.xpack.core.ilm.LifecycleSettings;
+import org.elasticsearch.xpack.core.ilm.OperationMode;
+import org.elasticsearch.xpack.core.ilm.Phase;
+import org.elasticsearch.xpack.core.ilm.RolloverAction;
+import org.elasticsearch.xpack.core.ilm.SetPriorityAction;
+import org.elasticsearch.xpack.core.ilm.ShrinkAction;
+import org.junit.AfterClass;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import static java.util.Collections.singletonMap;
+import static org.elasticsearch.xpack.TimeSeriesRestDriver.createIndexWithSettings;
+import static org.elasticsearch.xpack.TimeSeriesRestDriver.createNewSingletonPolicy;
+import static org.elasticsearch.xpack.TimeSeriesRestDriver.createPolicy;
+import static org.elasticsearch.xpack.TimeSeriesRestDriver.getOnlyIndexSettings;
+import static org.elasticsearch.xpack.TimeSeriesRestDriver.getStepKeyForIndex;
+import static org.hamcrest.Matchers.anEmptyMap;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+
+public class MigrateToDataTiersIT extends ESRestTestCase {
+    private static final Logger logger = LogManager.getLogger(MigrateToDataTiersIT.class);
+
+    private String index;
+    private String policy;
+    private String alias;
+
+    @Before
+    public void refreshIndexAndStartILM() throws IOException {
+        index = "index-" + randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
+        policy = "policy-" + randomAlphaOfLength(5);
+        alias = "alias-" + randomAlphaOfLength(5);
+        assertOK(client().performRequest(new Request("POST", "_ilm/start")));
+    }
+
+    @AfterClass
+    public static void restartIlm() throws IOException {
+        // some tests might stop ILM in order to perform the migration to data tiers, let's restart it
+        assertOK(client().performRequest(new Request("POST", "_ilm/start")));
+    }
+
+    public void testAPIFailsIfILMIsNotStopped() throws IOException {
+        Request migrateRequest = new Request("POST", "_ilm/migrate_to_data_tiers");
+        ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(migrateRequest));
+        assertThat(e.getResponse().getStatusLine().getStatusCode(), is(RestStatus.INTERNAL_SERVER_ERROR.getStatus()));
+        assertThat(e.getMessage(), containsString("stop ILM before migrating to data tiers, current state is [RUNNING]"));
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testMigrateToDataTiersAction() throws Exception {
+        // creating a legacy template to use in the migrate API
+        String templateName = randomAlphaOfLengthBetween(10, 15).toLowerCase(Locale.ROOT);
+        createLegacyTemplate(templateName);
+
+        // let's create a policy that'll need migrating, with a long `min_age` for the cold phase such that managed indices stop in
+        // Warm/Complete/Complete - this will ensure the migration will have to update the cached phase for these indices
+        Map<String, LifecycleAction> hotActions = new HashMap<>();
+        hotActions.put(SetPriorityAction.NAME, new SetPriorityAction(100));
+        Map<String, LifecycleAction> warmActions = new HashMap<>();
+        warmActions.put(SetPriorityAction.NAME, new SetPriorityAction(50));
+        warmActions.put(ForceMergeAction.NAME, new ForceMergeAction(1, null));
+        warmActions.put(AllocateAction.NAME, new AllocateAction(null, singletonMap("data", "warm"), null, null));
+        warmActions.put(ShrinkAction.NAME, new ShrinkAction(1, null));
+        Map<String, LifecycleAction> coldActions = new HashMap<>();
+        coldActions.put(SetPriorityAction.NAME, new SetPriorityAction(0));
+        coldActions.put(AllocateAction.NAME, new AllocateAction(0, null, null, singletonMap("data", "cold")));
+
+        createPolicy(client(), policy,
+            new Phase("hot", TimeValue.ZERO, hotActions),
+            new Phase("warm", TimeValue.ZERO, warmActions),
+            new Phase("cold", TimeValue.timeValueDays(100), coldActions),
+            null,
+            new Phase("delete", TimeValue.ZERO, singletonMap(DeleteAction.NAME, new DeleteAction()))
+        );
+
+        createIndexWithSettings(client(), index, alias, Settings.builder()
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put(LifecycleSettings.LIFECYCLE_NAME, policy)
+            .putNull(DataTierAllocationDecider.INDEX_ROUTING_PREFER)
+            .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias)
+        );
+
+        // wait for the index to advance to the warm phase
+        assertBusy(() ->
+            assertThat(getStepKeyForIndex(client(), index).getPhase(), equalTo("warm")), 30, TimeUnit.SECONDS);
+
+        // let's also have a policy that doesn't need migrating
+        String rolloverOnlyPolicyName = "rollover-policy";
+        createNewSingletonPolicy(client(), rolloverOnlyPolicyName, "hot", new RolloverAction(null, null, null, 1L));
+
+        String rolloverIndexPrefix = "rolloverpolicytest_index";
+        for (int i = 1; i < randomIntBetween(2, 5); i++) {
+            // assign the rollover-only policy to a few other indices - these indices and the rollover-only policy should not be migrated
+            // in any way
+            createIndexWithSettings(client(), rolloverIndexPrefix + "-00000" + i, alias + i, Settings.builder()
+                .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                .putNull(DataTierAllocationDecider.INDEX_ROUTING_PREFER)
+                .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias + i)
+            );
+        }
+
+        // let's stop ILM so we can perform the migration
+        client().performRequest(new Request("POST", "_ilm/stop"));
+        assertBusy(() -> {
+            Response response = client().performRequest(new Request("GET", "_ilm/status"));
+            assertThat(EntityUtils.toString(response.getEntity()), containsString(OperationMode.STOPPED.toString()));
+        });
+
+        String indexWithDataWarmRouting = "indexwithdatawarmrouting";
+        Settings.Builder settings = Settings.builder()
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put(IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + "data", "warm");
+        createIndex(indexWithDataWarmRouting, settings.build());
+
+        Request migrateRequest = new Request("POST", "_ilm/migrate_to_data_tiers");
+        migrateRequest.setJsonEntity(
+            "{\"legacy_template_to_delete\": \"" + templateName + "\", \"node_attribute\": \"data\"}"
+        );
+        Response migrateDeploymentResponse = client().performRequest(migrateRequest);
+        assertOK(migrateDeploymentResponse);
+
+        Map<String, Object> migrateResponseAsMap = responseAsMap(migrateDeploymentResponse);
+        assertThat((ArrayList<String>) migrateResponseAsMap.get(MigrateToDataTiersResponse.MIGRATED_ILM_POLICIES.getPreferredName()),
+            containsInAnyOrder(policy));
+        assertThat((ArrayList<String>) migrateResponseAsMap.get(MigrateToDataTiersResponse.MIGRATED_INDICES.getPreferredName()),
+            containsInAnyOrder(index, indexWithDataWarmRouting));
+        assertThat(migrateResponseAsMap.get(MigrateToDataTiersResponse.REMOVED_LEGACY_TEMPLATE.getPreferredName()),
+            is(templateName));
+
+        // let's verify the legacy template doesn't exist anymore
+        Request getTemplateRequest = new Request("HEAD", "_template/" + templateName);
+        assertThat(client().performRequest(getTemplateRequest).getStatusLine().getStatusCode(), is(RestStatus.NOT_FOUND.getStatus()));
+
+        // let's assert the require.data:warm configuration the "indexWithDataWarmRouting" had was migrated to
+        // _tier_preference:data_warm,data_hot
+        Map<String, Object> indexSettings = getOnlyIndexSettings(client(), indexWithDataWarmRouting);
+        assertThat(indexSettings.get(DataTierAllocationDecider.INDEX_ROUTING_PREFER), is("data_warm,data_hot"));
+
+        // let's retrieve the migrated policy and check it was migrated correctly - namely the warm phase should not contain any allocate
+        // action anymore and the cold phase should contain an allocate action that only configures the number of replicas
+        Request getPolicy = new Request("GET", "/_ilm/policy/" + policy);
+        Map<String, Object> policyAsMap = (Map<String, Object>) responseAsMap(client().performRequest(getPolicy)).get(policy);
+        Map<String, Object> warmActionsMap = getActionsForPhase(policyAsMap, "warm");
+        assertThat(warmActionsMap.size(), is(3));
+        assertThat(warmActionsMap.get(AllocateAction.NAME), nullValue());
+        Map<String, Object> coldActionsMap = getActionsForPhase(policyAsMap, "cold");
+        assertThat(coldActionsMap.size(), is(2));
+        assertThat(coldActionsMap.get(AllocateAction.NAME), notNullValue());
+        Map<String, Object> coldAllocateActionMap = (Map<String, Object>) coldActionsMap.get(AllocateAction.NAME);
+        assertThat((Map<String, Object>) coldAllocateActionMap.get("include"), is(anEmptyMap()));
+        assertThat((Map<String, Object>) coldAllocateActionMap.get("require"), is(anEmptyMap()));
+        assertThat((Map<String, Object>) coldAllocateActionMap.get("exclude"), is(anEmptyMap()));
+
+        Request getClusterMetadataRequest = new Request("GET", "/_cluster/state/metadata/" + index);
+        Response clusterMetadataResponse = client().performRequest(getClusterMetadataRequest);
+
+        String cachedPhaseDefinition = getCachedPhaseDefAsMap(clusterMetadataResponse, index);
+        // let's also verify the cached phase definition was updated - as the managed index was in the warm phase, which after migration
+        // does not contain the allocate action anymore, the cached warm phase should not contain the allocate action either
+        assertThat("the cached phase definition should reflect the migrated warm phase which must NOT contain an allocate action anymore",
+            cachedPhaseDefinition, not(containsString(AllocateAction.NAME)));
+        assertThat(cachedPhaseDefinition, containsString(ShrinkAction.NAME));
+        assertThat(cachedPhaseDefinition, containsString(SetPriorityAction.NAME));
+        assertThat(cachedPhaseDefinition, containsString(ForceMergeAction.NAME));
+    }
+
+    @SuppressWarnings("unchecked")
+    private Map<String, Object> getActionsForPhase(Map<String, Object> policyAsMap, String phase) {
+        Map<String, Object> phases = (Map<String, Object>) ((Map<String, Object>) policyAsMap.get("policy")).get("phases");
+        return (Map<String, Object>) ((Map<String, Object>) phases.get(phase)).get("actions");
+    }
+
+    @SuppressWarnings("unchecked")
+    private String getCachedPhaseDefAsMap(Response clusterMetadataResponse, String indexName) throws IOException {
+        Map<String, Object> clusterMetadataMap = responseAsMap(clusterMetadataResponse);
+        Map<String, Object> metadata = (Map<String, Object>) clusterMetadataMap.get("metadata");
+        Map<String, Object> indices = (Map<String, Object>) metadata.get("indices");
+        Map<String, Object> indexMetadata = (Map<String, Object>) indices.get(indexName);
+        Map<String, Object> ilmMetadata = (Map<String, Object>) indexMetadata.get("ilm");
+        return (String) ilmMetadata.get("phase_definition");
+    }
+
+    private void createLegacyTemplate(String templateName) throws IOException {
+        String indexPrefix = randomAlphaOfLengthBetween(5, 15).toLowerCase(Locale.ROOT);
+        final StringEntity template = new StringEntity("{\n" +
+            "  \"index_patterns\": \"" + indexPrefix + "*\",\n" +
+            "  \"settings\": {\n" +
+            "    \"index\": {\n" +
+            "      \"lifecycle\": {\n" +
+            "        \"name\": \"does_not_exist\",\n" +
+            "        \"rollover_alias\": \"test_alias\"\n" +
+            "      }\n" +
+            "    }\n" +
+            "  }\n" +
+            "}", ContentType.APPLICATION_JSON);
+        Request templateRequest = new Request("PUT", "_template/" + templateName);
+        templateRequest.setEntity(template);
+        templateRequest.setOptions(expectWarnings(RestPutIndexTemplateAction.DEPRECATION_WARNING));
+        client().performRequest(templateRequest);
+    }
+}

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java

@@ -16,7 +16,6 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.xcontent.ParseField;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry;
 import org.elasticsearch.common.settings.ClusterSettings;
@@ -25,6 +24,7 @@ import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ParseField;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
@@ -40,6 +40,9 @@ import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.watcher.ResourceWatcherService;
 import org.elasticsearch.xpack.core.XPackPlugin;
+import org.elasticsearch.xpack.cluster.action.MigrateToDataTiersAction;
+import org.elasticsearch.xpack.ilm.action.TransportMigrateToDataTiersAction;
+import org.elasticsearch.xpack.ilm.action.RestMigrateToDataTiersAction;
 import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
 import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
 import org.elasticsearch.xpack.core.ilm.AllocateAction;
@@ -259,8 +262,8 @@ public class IndexLifecycle extends Plugin implements ActionPlugin {
             IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver,
             Supplier<DiscoveryNodes> nodesInCluster) {
         List<RestHandler> handlers = new ArrayList<>();
-        handlers.addAll(Arrays.asList(
 
+        handlers.addAll(Arrays.asList(
             // add ILM rest handlers
             new RestPutLifecycleAction(),
             new RestGetLifecycleAction(),
@@ -272,6 +275,7 @@ public class IndexLifecycle extends Plugin implements ActionPlugin {
             new RestStopAction(),
             new RestStartILMAction(),
             new RestGetStatusAction(),
+            new RestMigrateToDataTiersAction(),
 
             // add SLM rest headers
             new RestPutSnapshotLifecycleAction(),
@@ -293,11 +297,13 @@ public class IndexLifecycle extends Plugin implements ActionPlugin {
         var ilmInfoAction = new ActionHandler<>(XPackInfoFeatureAction.INDEX_LIFECYCLE, IndexLifecycleInfoTransportAction.class);
         var slmUsageAction = new ActionHandler<>(XPackUsageFeatureAction.SNAPSHOT_LIFECYCLE, SLMUsageTransportAction.class);
         var slmInfoAction = new ActionHandler<>(XPackInfoFeatureAction.SNAPSHOT_LIFECYCLE, SLMInfoTransportAction.class);
+        var migrateToDataTiersAction = new ActionHandler<>(MigrateToDataTiersAction.INSTANCE, TransportMigrateToDataTiersAction.class);
         List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> actions = new ArrayList<>();
         actions.add(ilmUsageAction);
         actions.add(ilmInfoAction);
         actions.add(slmUsageAction);
         actions.add(slmInfoAction);
+        actions.add(migrateToDataTiersAction);
         actions.addAll(Arrays.asList(
             // add ILM actions
             new ActionHandler<>(PutLifecycleAction.INSTANCE, TransportPutLifecycleAction.class),

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestMigrateToDataTiersAction.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.ilm.action;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.xpack.cluster.action.MigrateToDataTiersAction;
+import org.elasticsearch.xpack.cluster.action.MigrateToDataTiersRequest;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.elasticsearch.rest.RestRequest.Method.POST;
+
+public class RestMigrateToDataTiersAction extends BaseRestHandler {
+
+    @Override
+    public String getName() {
+        return "migrate_to_data_tiers_action";
+    }
+
+    @Override
+    public List<Route> routes() {
+        return List.of(new Route(POST, "/_ilm/migrate_to_data_tiers"));
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+        MigrateToDataTiersRequest migrateRequest = request.hasContent() ?
+            MigrateToDataTiersRequest.parse(request.contentParser()) : new MigrateToDataTiersRequest();
+        return channel -> client.execute(MigrateToDataTiersAction.INSTANCE, migrateRequest, new RestToXContentListener<>(channel));
+    }
+}

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportMigrateToDataTiersAction.java

@@ -0,0 +1,98 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.ilm.action;
+
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.master.TransportMasterNodeAction;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Priority;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.cluster.action.MigrateToDataTiersAction;
+import org.elasticsearch.xpack.cluster.action.MigrateToDataTiersRequest;
+import org.elasticsearch.xpack.cluster.action.MigrateToDataTiersResponse;
+import org.elasticsearch.xpack.cluster.metadata.MetadataMigrateToDataTiersRoutingService.MigratedEntities;
+import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata;
+
+import static org.elasticsearch.xpack.cluster.metadata.MetadataMigrateToDataTiersRoutingService.migrateToDataTiersRouting;
+import static org.elasticsearch.xpack.core.ilm.OperationMode.STOPPED;
+
+public class TransportMigrateToDataTiersAction extends TransportMasterNodeAction<MigrateToDataTiersRequest, MigrateToDataTiersResponse> {
+
+    private final NamedXContentRegistry xContentRegistry;
+    private final Client client;
+    private final XPackLicenseState licenseState;
+
+    @Inject
+    public TransportMigrateToDataTiersAction(TransportService transportService, ClusterService clusterService,
+                                             ThreadPool threadPool, ActionFilters actionFilters,
+                                             IndexNameExpressionResolver indexNameExpressionResolver,
+                                             NamedXContentRegistry xContentRegistry, Client client, XPackLicenseState licenseState) {
+        super(MigrateToDataTiersAction.NAME, transportService, clusterService, threadPool, actionFilters, MigrateToDataTiersRequest::new,
+            indexNameExpressionResolver, MigrateToDataTiersResponse::new, ThreadPool.Names.SAME);
+        this.xContentRegistry = xContentRegistry;
+        this.client = client;
+        this.licenseState = licenseState;
+    }
+
+    @Override
+    protected void masterOperation(Task task, MigrateToDataTiersRequest request, ClusterState state,
+                                   ActionListener<MigrateToDataTiersResponse> listener) throws Exception {
+        IndexLifecycleMetadata currentMetadata = state.metadata().custom(IndexLifecycleMetadata.TYPE);
+        if (currentMetadata != null && currentMetadata.getOperationMode() != STOPPED) {
+            listener.onFailure(new IllegalStateException("stop ILM before migrating to data tiers, current state is [" +
+                currentMetadata.getOperationMode() + "]"));
+            return;
+        }
+        final SetOnce<MigratedEntities> migratedEntities = new SetOnce<>();
+        clusterService.submitStateUpdateTask("migrate-to-data-tiers []", new ClusterStateUpdateTask(Priority.HIGH) {
+            @Override
+            public ClusterState execute(ClusterState currentState) throws Exception {
+                Tuple<ClusterState, MigratedEntities> migratedEntitiesTuple =
+                    migrateToDataTiersRouting(state, request.getNodeAttributeName(), request.getLegacyTemplateToDelete(),
+                        xContentRegistry, client, licenseState);
+
+                migratedEntities.set(migratedEntitiesTuple.v2());
+                return migratedEntitiesTuple.v1();
+            }
+
+            @Override
+            public void onFailure(String source, Exception e) {
+                listener.onFailure(e);
+            }
+
+            @Override
+            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                super.clusterStateProcessed(source, oldState, newState);
+                MigratedEntities entities = migratedEntities.get();
+                listener.onResponse(
+                    new MigrateToDataTiersResponse(entities.removedIndexTemplateName, entities.migratedPolicies, entities.migratedIndices)
+                );
+            }
+        });
+
+    }
+
+    @Override
+    protected ClusterBlockException checkBlock(MigrateToDataTiersRequest request, ClusterState state) {
+        return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
+    }
+}

x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java

@@ -44,6 +44,7 @@ public class Constants {
         "cluster:admin/logstash/pipeline/delete",
         "cluster:admin/logstash/pipeline/get",
         "cluster:admin/logstash/pipeline/put",
+        "cluster:admin/migrate_to_data_tiers",
         "cluster:admin/nodes/reload_secure_settings",
         "cluster:admin/persistent/completion",
         "cluster:admin/persistent/remove",