Add `dry_run` support to the migrate to data tiers API (#74639)

This adds support for a `dry_run` parameter for the
`_ilm/migrate_to_data_tiers` API. This defaults to `false`, but when
configured to `true` it will simulate the migration of elasticsearch
entities to data tiers based routing, returning the entites that need to
be updated (indices, ILM policies and the legacy index template that'd
be deleted, if any was configured in the request).

docs/reference/ilm/apis/migrate-to-data-tiers.asciidoc

@@ -41,6 +41,16 @@ Defaults to `data`.
 to stop {ilm-init} and <<ilm-get-status-request, get status API>> to wait until the
 reported operation mode is `STOPPED`.
 
+[[ilm-migrate-to-data-tiers-query-params]]
+==== {api-query-parms-title}
+
+`dry_run`::
+(Optional, Boolean)
+If `true`, simulates the migration from node attributes based allocation filters to data tiers, but does
+not perform the migration. This provides a way to retrieve the indices and ILM policies that need to be
+migrated.
+Defaults to `false`.
+
 [[ilm-migrate-to-data-tiers-example]]
 ==== {api-examples-title}
 
@@ -118,6 +128,7 @@ If the request succeeds, a response like the following will be received:
 [source,console-result]
 ------------------------------------------------------------------------------
 {
+  "dry_run": false,
   "removed_legacy_template":"global-template", <1>
   "migrated_ilm_policies":["policy_with_allocate_action"], <2>
   "migrated_indices":["warm-index-to-migrate-000001"] <3>

rest-api-spec/src/main/resources/rest-api-spec/api/ilm.migrate_to_data_tiers.json

@@ -20,7 +20,12 @@
         }
       ]
     },
-    "params":{},
+    "params": {
+      "dry_run": {
+        "type": "boolean",
+        "description": "If set to true it will simulate the migration, providing a way to retrieve the ILM policies and indices that need to be migrated. The default is false"
+      }
+    },
     "body":{
       "description":"Optionally specify a legacy index template name to delete and optionally specify a node attribute name used for index shard routing (defaults to \"data\")",
       "required":false

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/action/MigrateToDataTiersRequest.java

@@ -45,6 +45,7 @@ public class MigrateToDataTiersRequest extends AcknowledgedRequest<MigrateToData
      */
     @Nullable
     private final String legacyTemplateToDelete;
+    private boolean dryRun = false;
 
     public static MigrateToDataTiersRequest parse(XContentParser parser) throws IOException {
         return PARSER.parse(parser, null);
@@ -61,6 +62,7 @@ public class MigrateToDataTiersRequest extends AcknowledgedRequest<MigrateToData
 
     public MigrateToDataTiersRequest(StreamInput in) throws IOException {
         super(in);
+        dryRun = in.readBoolean();
         legacyTemplateToDelete = in.readOptionalString();
         nodeAttributeName = in.readOptionalString();
     }
@@ -73,10 +75,15 @@ public class MigrateToDataTiersRequest extends AcknowledgedRequest<MigrateToData
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
+        out.writeBoolean(dryRun);
         out.writeOptionalString(legacyTemplateToDelete);
         out.writeOptionalString(nodeAttributeName);
     }
 
+    public void setDryRun(boolean dryRun) {
+        this.dryRun = dryRun;
+    }
+
     public String getNodeAttributeName() {
         return nodeAttributeName;
     }
@@ -85,6 +92,10 @@ public class MigrateToDataTiersRequest extends AcknowledgedRequest<MigrateToData
         return legacyTemplateToDelete;
     }
 
+    public boolean isDryRun() {
+        return dryRun;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) {
@@ -94,20 +105,13 @@ public class MigrateToDataTiersRequest extends AcknowledgedRequest<MigrateToData
             return false;
         }
         MigrateToDataTiersRequest that = (MigrateToDataTiersRequest) o;
-        return Objects.equals(nodeAttributeName, that.nodeAttributeName) && Objects.equals(legacyTemplateToDelete,
-            that.legacyTemplateToDelete);
+        return dryRun == that.dryRun &&
+            Objects.equals(nodeAttributeName, that.nodeAttributeName) &&
+            Objects.equals(legacyTemplateToDelete, that.legacyTemplateToDelete);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(nodeAttributeName, legacyTemplateToDelete);
-    }
-
-    @Override
-    public String toString() {
-        return "MigrateToDataTiersRequest{" +
-            "nodeAttributeName='" + nodeAttributeName + '\'' +
-            ", legacyTemplateToDelete='" + legacyTemplateToDelete + '\'' +
-            '}';
+        return Objects.hash(nodeAttributeName, legacyTemplateToDelete, dryRun);
     }
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/action/MigrateToDataTiersResponse.java

@@ -17,23 +17,27 @@ import org.elasticsearch.core.Nullable;
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Objects;
 
 public class MigrateToDataTiersResponse extends ActionResponse implements ToXContentObject {
 
     public static final ParseField REMOVED_LEGACY_TEMPLATE = new ParseField("removed_legacy_template");
     public static final ParseField MIGRATED_INDICES = new ParseField("migrated_indices");
     public static final ParseField MIGRATED_ILM_POLICIES = new ParseField("migrated_ilm_policies");
+    private static final ParseField DRY_RUN = new ParseField("dry_run");
 
     @Nullable
     private final String removedIndexTemplateName;
     private final List<String> migratedPolicies;
     private final List<String> migratedIndices;
+    private final boolean dryRun;
 
     public MigrateToDataTiersResponse(@Nullable String removedIndexTemplateName, List<String> migratedPolicies,
-                                      List<String> migratedIndices) {
+                                      List<String> migratedIndices, boolean dryRun) {
         this.removedIndexTemplateName = removedIndexTemplateName;
         this.migratedPolicies = migratedPolicies;
         this.migratedIndices = migratedIndices;
+        this.dryRun = dryRun;
     }
 
     public MigrateToDataTiersResponse(StreamInput in) throws IOException {
@@ -41,11 +45,13 @@ public class MigrateToDataTiersResponse extends ActionResponse implements ToXCon
         removedIndexTemplateName = in.readOptionalString();
         migratedPolicies = in.readStringList();
         migratedIndices = in.readStringList();
+        dryRun = in.readBoolean();
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
+        builder.field(DRY_RUN.getPreferredName(), dryRun);
         if (this.removedIndexTemplateName != null) {
             builder.field(REMOVED_LEGACY_TEMPLATE.getPreferredName(), this.removedIndexTemplateName);
         }
@@ -67,10 +73,45 @@ public class MigrateToDataTiersResponse extends ActionResponse implements ToXCon
         return builder;
     }
 
+    public String getRemovedIndexTemplateName() {
+        return removedIndexTemplateName;
+    }
+
+    public List<String> getMigratedPolicies() {
+        return migratedPolicies;
+    }
+
+    public List<String> getMigratedIndices() {
+        return migratedIndices;
+    }
+
+    public boolean isDryRun() {
+        return dryRun;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(removedIndexTemplateName);
         out.writeStringCollection(migratedPolicies);
         out.writeStringCollection(migratedIndices);
+        out.writeBoolean(dryRun);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        MigrateToDataTiersResponse that = (MigrateToDataTiersResponse) o;
+        return dryRun == that.dryRun && Objects.equals(removedIndexTemplateName, that.removedIndexTemplateName) &&
+            Objects.equals(migratedPolicies, that.migratedPolicies) && Objects.equals(migratedIndices, that.migratedIndices);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(removedIndexTemplateName, migratedPolicies, migratedIndices, dryRun);
     }
 }

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/action/MigrateToDataTiersResponseTests.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.cluster.action;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+
+import java.io.IOException;
+
+public class MigrateToDataTiersResponseTests extends AbstractWireSerializingTestCase<MigrateToDataTiersResponse> {
+
+    @Override
+    protected Writeable.Reader<MigrateToDataTiersResponse> instanceReader() {
+        return MigrateToDataTiersResponse::new;
+    }
+
+    @Override
+    protected MigrateToDataTiersResponse createTestInstance() {
+        boolean dryRun = randomBoolean();
+        return new MigrateToDataTiersResponse(randomAlphaOfLength(10), randomList(1, 5, () -> randomAlphaOfLengthBetween(5, 50)),
+            randomList(1, 5, () -> randomAlphaOfLengthBetween(5, 50)), dryRun);
+    }
+
+    @Override
+    protected MigrateToDataTiersResponse mutateInstance(MigrateToDataTiersResponse instance) throws IOException {
+        int i = randomIntBetween(0, 3);
+        switch (i) {
+            case 0:
+                return new MigrateToDataTiersResponse(randomValueOtherThan(instance.getRemovedIndexTemplateName(),
+                    () -> randomAlphaOfLengthBetween(5, 15)), instance.getMigratedPolicies(), instance.getMigratedIndices(),
+                    instance.isDryRun());
+            case 1:
+                return new MigrateToDataTiersResponse(instance.getRemovedIndexTemplateName(),
+                    randomList(6, 10, () -> randomAlphaOfLengthBetween(5, 50)), instance.getMigratedIndices(), instance.isDryRun());
+            case 2:
+                return new MigrateToDataTiersResponse(instance.getRemovedIndexTemplateName(), instance.getMigratedPolicies(),
+                    randomList(6, 10, () -> randomAlphaOfLengthBetween(5, 50)), instance.isDryRun());
+            case 3:
+                return new MigrateToDataTiersResponse(instance.getRemovedIndexTemplateName(), instance.getMigratedPolicies(),
+                    instance.getMigratedIndices(), instance.isDryRun() ? false : true);
+            default:
+                throw new UnsupportedOperationException();
+        }
+    }
+}

x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java

@@ -24,6 +24,7 @@ import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.xpack.cluster.action.MigrateToDataTiersResponse;
 import org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDecider;
 import org.elasticsearch.xpack.core.ilm.AllocateAction;
+import org.elasticsearch.xpack.core.ilm.AllocationRoutedStep;
 import org.elasticsearch.xpack.core.ilm.DeleteAction;
 import org.elasticsearch.xpack.core.ilm.ForceMergeAction;
 import org.elasticsearch.xpack.core.ilm.LifecycleAction;
@@ -124,6 +125,9 @@ public class MigrateToDataTiersIT extends ESRestTestCase {
         // wait for the index to advance to the warm phase
         assertBusy(() ->
             assertThat(getStepKeyForIndex(client(), index).getPhase(), equalTo("warm")), 30, TimeUnit.SECONDS);
+        // let's wait for this index to have received the `require.data` configuration from the warm phase/allocate action
+        assertBusy(() ->
+            assertThat(getStepKeyForIndex(client(), index).getName(), equalTo(AllocationRoutedStep.NAME)), 30, TimeUnit.SECONDS);
 
         // let's also have a policy that doesn't need migrating
         String rolloverOnlyPolicyName = "rollover-policy";
@@ -207,6 +211,93 @@ public class MigrateToDataTiersIT extends ESRestTestCase {
         assertThat(cachedPhaseDefinition, containsString(ForceMergeAction.NAME));
     }
 
+    @SuppressWarnings("unchecked")
+    public void testMigrationDryRun() throws Exception {
+        String templateName = randomAlphaOfLengthBetween(10, 15).toLowerCase(Locale.ROOT);
+        createLegacyTemplate(templateName);
+
+        Map<String, LifecycleAction> hotActions = new HashMap<>();
+        hotActions.put(SetPriorityAction.NAME, new SetPriorityAction(100));
+        Map<String, LifecycleAction> warmActions = new HashMap<>();
+        warmActions.put(SetPriorityAction.NAME, new SetPriorityAction(50));
+        warmActions.put(ForceMergeAction.NAME, new ForceMergeAction(1, null));
+        warmActions.put(AllocateAction.NAME, new AllocateAction(null, singletonMap("data", "warm"), null, null));
+        warmActions.put(ShrinkAction.NAME, new ShrinkAction(1, null));
+        Map<String, LifecycleAction> coldActions = new HashMap<>();
+        coldActions.put(SetPriorityAction.NAME, new SetPriorityAction(0));
+        coldActions.put(AllocateAction.NAME, new AllocateAction(0, null, null, singletonMap("data", "cold")));
+
+        createPolicy(client(), policy,
+            new Phase("hot", TimeValue.ZERO, hotActions),
+            new Phase("warm", TimeValue.ZERO, warmActions),
+            new Phase("cold", TimeValue.timeValueDays(100), coldActions),
+            null,
+            new Phase("delete", TimeValue.ZERO, singletonMap(DeleteAction.NAME, new DeleteAction()))
+        );
+
+        createIndexWithSettings(client(), index, alias, Settings.builder()
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put(LifecycleSettings.LIFECYCLE_NAME, policy)
+            .putNull(DataTierAllocationDecider.INDEX_ROUTING_PREFER)
+            .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias)
+        );
+
+        // wait for the index to advance to the warm phase
+        assertBusy(() ->
+            assertThat(getStepKeyForIndex(client(), index).getPhase(), equalTo("warm")), 30, TimeUnit.SECONDS);
+        // let's wait for this index to have received the `require.data` configuration from the warm phase/allocate action
+        assertBusy(() ->
+            assertThat(getStepKeyForIndex(client(), index).getName(), equalTo(AllocationRoutedStep.NAME)), 30, TimeUnit.SECONDS);
+
+        // let's stop ILM so we can simulate the migration
+        client().performRequest(new Request("POST", "_ilm/stop"));
+        assertBusy(() -> {
+            Response response = client().performRequest(new Request("GET", "_ilm/status"));
+            assertThat(EntityUtils.toString(response.getEntity()), containsString(OperationMode.STOPPED.toString()));
+        });
+
+        String indexWithDataWarmRouting = "indexwithdatawarmrouting";
+        Settings.Builder settings = Settings.builder()
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put(IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + "data", "warm");
+        createIndex(indexWithDataWarmRouting, settings.build());
+
+        Request migrateRequest = new Request("POST", "_ilm/migrate_to_data_tiers");
+        migrateRequest.addParameter("dry_run", "true");
+        migrateRequest.setJsonEntity(
+            "{\"legacy_template_to_delete\": \"" + templateName + "\", \"node_attribute\": \"data\"}"
+        );
+        Response migrateDeploymentResponse = client().performRequest(migrateRequest);
+        assertOK(migrateDeploymentResponse);
+
+        // response should contain the correct "to migrate" entities
+        Map<String, Object> migrateResponseAsMap = responseAsMap(migrateDeploymentResponse);
+        assertThat((ArrayList<String>) migrateResponseAsMap.get(MigrateToDataTiersResponse.MIGRATED_ILM_POLICIES.getPreferredName()),
+            containsInAnyOrder(policy));
+        assertThat((ArrayList<String>) migrateResponseAsMap.get(MigrateToDataTiersResponse.MIGRATED_INDICES.getPreferredName()),
+            containsInAnyOrder(index, indexWithDataWarmRouting));
+        assertThat(migrateResponseAsMap.get(MigrateToDataTiersResponse.REMOVED_LEGACY_TEMPLATE.getPreferredName()),
+            is(templateName));
+
+        // however the entities should NOT have been changed
+        // the index template should still exist
+        Request getTemplateRequest = new Request("HEAD", "_template/" + templateName);
+        assertThat(client().performRequest(getTemplateRequest).getStatusLine().getStatusCode(), is(RestStatus.OK.getStatus()));
+
+        // the index settings should not contain the _tier_preference
+        Map<String, Object> indexSettings = getOnlyIndexSettings(client(), indexWithDataWarmRouting);
+        assertThat(indexSettings.get(DataTierAllocationDecider.INDEX_ROUTING_PREFER), nullValue());
+
+        // let's check the ILM policy was not migrated - ie. the warm phase still contains the allocate action
+        Request getPolicy = new Request("GET", "/_ilm/policy/" + policy);
+        Map<String, Object> policyAsMap = (Map<String, Object>) responseAsMap(client().performRequest(getPolicy)).get(policy);
+        Map<String, Object> warmActionsMap = getActionsForPhase(policyAsMap, "warm");
+        assertThat(warmActionsMap.size(), is(4));
+        assertThat(warmActionsMap.get(AllocateAction.NAME), notNullValue());
+    }
+
     @SuppressWarnings("unchecked")
     private Map<String, Object> getActionsForPhase(Map<String, Object> policyAsMap, String phase) {
         Map<String, Object> phases = (Map<String, Object>) ((Map<String, Object>) policyAsMap.get("policy")).get("phases");

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestMigrateToDataTiersAction.java

@@ -35,6 +35,7 @@ public class RestMigrateToDataTiersAction extends BaseRestHandler {
     protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
         MigrateToDataTiersRequest migrateRequest = request.hasContent() ?
             MigrateToDataTiersRequest.parse(request.contentParser()) : new MigrateToDataTiersRequest();
+        migrateRequest.setDryRun(request.paramAsBoolean("dry_run", false));
         return channel -> client.execute(MigrateToDataTiersAction.INSTANCE, migrateRequest, new RestToXContentListener<>(channel));
     }
 }

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportMigrateToDataTiersAction.java

@@ -62,6 +62,17 @@ public class TransportMigrateToDataTiersAction extends TransportMasterNodeAction
                 currentMetadata.getOperationMode() + "]"));
             return;
         }
+
+        if (request.isDryRun()) {
+            MigratedEntities entities =
+                migrateToDataTiersRouting(state, request.getNodeAttributeName(), request.getLegacyTemplateToDelete(),
+                    xContentRegistry, client, licenseState).v2();
+            listener.onResponse(
+                new MigrateToDataTiersResponse(entities.removedIndexTemplateName, entities.migratedPolicies, entities.migratedIndices, true)
+            );
+            return;
+        }
+
         final SetOnce<MigratedEntities> migratedEntities = new SetOnce<>();
         clusterService.submitStateUpdateTask("migrate-to-data-tiers []", new ClusterStateUpdateTask(Priority.HIGH) {
             @Override
@@ -83,8 +94,8 @@ public class TransportMigrateToDataTiersAction extends TransportMasterNodeAction
             public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                 super.clusterStateProcessed(source, oldState, newState);
                 MigratedEntities entities = migratedEntities.get();
-                listener.onResponse(
-                    new MigrateToDataTiersResponse(entities.removedIndexTemplateName, entities.migratedPolicies, entities.migratedIndices)
+                listener.onResponse(new MigrateToDataTiersResponse(entities.removedIndexTemplateName, entities.migratedPolicies,
+                    entities.migratedIndices, false)
                 );
             }
         });