Improve error message for deleting in-use policy (#36457)

The error message used when attempting to delete a lifecycle policy that
is in use previously only included one index which was using the policy.
It now includes all indices using that policy.

x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java

@@ -35,6 +35,7 @@ import org.elasticsearch.xpack.core.indexlifecycle.ShrinkStep;
 import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
 import org.elasticsearch.xpack.core.indexlifecycle.TerminalPolicyStep;
 import org.elasticsearch.xpack.core.indexlifecycle.WaitForRolloverReadyStep;
+import org.hamcrest.Matchers;
 import org.junit.Before;
 
 import java.io.IOException;
@@ -495,6 +496,41 @@ public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
         assertThat(ex.getMessage(), containsString("invalid policy name"));
     }
 
+    public void testDeletePolicyInUse() throws IOException {
+        String managedIndex1 = randomAlphaOfLength(7).toLowerCase(Locale.ROOT);
+        String managedIndex2 = randomAlphaOfLength(8).toLowerCase(Locale.ROOT);
+        String unmanagedIndex = randomAlphaOfLength(9).toLowerCase(Locale.ROOT);
+        String managedByOtherPolicyIndex = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
+
+        createNewSingletonPolicy("delete", new DeleteAction(), TimeValue.timeValueHours(12));
+        String originalPolicy = policy;
+        String otherPolicy = randomValueOtherThan(policy, () -> randomAlphaOfLength(5));
+        policy = otherPolicy;
+        createNewSingletonPolicy("delete", new DeleteAction(), TimeValue.timeValueHours(13));
+
+        createIndexWithSettingsNoAlias(managedIndex1, Settings.builder()
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1,10))
+            .put(LifecycleSettings.LIFECYCLE_NAME_SETTING.getKey(), originalPolicy));
+        createIndexWithSettingsNoAlias(managedIndex2, Settings.builder()
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1,10))
+            .put(LifecycleSettings.LIFECYCLE_NAME_SETTING.getKey(), originalPolicy));
+        createIndexWithSettingsNoAlias(unmanagedIndex, Settings.builder()
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1,10)));
+        createIndexWithSettingsNoAlias(managedByOtherPolicyIndex, Settings.builder()
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1,10))
+            .put(LifecycleSettings.LIFECYCLE_NAME_SETTING.getKey(), otherPolicy));
+
+        Request deleteRequest = new Request("DELETE", "_ilm/policy/" + originalPolicy);
+        ResponseException ex = expectThrows(ResponseException.class, () -> client().performRequest(deleteRequest));
+        assertThat(ex.getCause().getMessage(),
+            Matchers.allOf(
+                containsString("Cannot delete policy [" + originalPolicy + "]. It is in use by one or more indices: ["),
+                containsString(managedIndex1),
+                containsString(managedIndex2),
+                not(containsString(unmanagedIndex)),
+                not(containsString(managedByOtherPolicyIndex))));
+    }
+
     private void createFullPolicy(TimeValue hotTime) throws IOException {
         Map<String, LifecycleAction> warmActions = new HashMap<>();
         warmActions.put(ForceMergeAction.NAME, new ForceMergeAction(1));
@@ -534,15 +570,23 @@ public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
         client().performRequest(request);
     }
 
+    private void createIndexWithSettingsNoAlias(String index, Settings.Builder settings) throws IOException {
+        Request request = new Request("PUT", "/" + index);
+        request.setJsonEntity("{\n \"settings\": " + Strings.toString(settings.build())
+            + "}");
+        client().performRequest(request);
+        // wait for the shards to initialize
+        ensureGreen(index);
+    }
+
     private void createIndexWithSettings(String index, Settings.Builder settings) throws IOException {
-        // create the test-index index
         Request request = new Request("PUT", "/" + index);
+
         request.setJsonEntity("{\n \"settings\": " + Strings.toString(settings.build())
             + ", \"aliases\" : { \"alias\": { \"is_write_index\": true } } }");
         client().performRequest(request);
         // wait for the shards to initialize
         ensureGreen(index);
-
     }
 
     private static void index(RestClient client, String index, String id, Object... fields) throws IOException {

x-pack/plugin/ilm/qa/rest/src/test/resources/rest-api-spec/test/ilm/10_basic.yml

@@ -202,7 +202,7 @@ setup:
       ilm.delete_lifecycle:
         policy: "my_timeseries_lifecycle"
   - match: { error.root_cause.0.type: "illegal_argument_exception" }
-  - match: { error.root_cause.0.reason: "Cannot delete policy [my_timeseries_lifecycle]. It is being used by at least one index [my_timeseries_index]" }
+  - match: { error.root_cause.0.reason: "Cannot delete policy [my_timeseries_lifecycle]. It is in use by one or more indices: [my_timeseries_index]" }
 
   - do:
       ilm.remove_policy:

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportDeleteLifecycleAction.java

@@ -6,6 +6,7 @@
 
 package org.elasticsearch.xpack.indexlifecycle.action;
 
+import com.carrotsearch.hppc.cursors.ObjectCursor;
 import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
@@ -23,14 +24,18 @@ import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicyMetadata;
-import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
 import org.elasticsearch.xpack.core.indexlifecycle.action.DeleteLifecycleAction;
 import org.elasticsearch.xpack.core.indexlifecycle.action.DeleteLifecycleAction.Request;
 import org.elasticsearch.xpack.core.indexlifecycle.action.DeleteLifecycleAction.Response;
 
-import java.util.Iterator;
+import java.util.List;
 import java.util.SortedMap;
+import java.util.Spliterator;
 import java.util.TreeMap;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+
+import static org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings.LIFECYCLE_NAME_SETTING;
 
 public class TransportDeleteLifecycleAction extends TransportMasterNodeAction<Request, Response> {
 
@@ -62,15 +67,16 @@ public class TransportDeleteLifecycleAction extends TransportMasterNodeAction<Re
 
                     @Override
                     public ClusterState execute(ClusterState currentState) {
-                        Iterator<IndexMetaData> indicesIt = currentState.metaData().indices().valuesIt();
-                        while(indicesIt.hasNext()) {
-                            IndexMetaData idxMeta = indicesIt.next();
-                            String indexPolicy = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(idxMeta.getSettings());
-                            if (request.getPolicyName().equals(indexPolicy)) {
-                                throw new IllegalArgumentException("Cannot delete policy [" + request.getPolicyName()
-                                    + "]. It is being used by at least one index [" + idxMeta.getIndex().getName() + "]");
-                            }
-
+                        Spliterator<ObjectCursor<IndexMetaData>> indicesIt = currentState.metaData().indices().values().spliterator();
+                        String policyToDelete = request.getPolicyName();
+                        List<String> indicesUsingPolicy = StreamSupport.stream(indicesIt, false)
+                            .map(idxMeta -> idxMeta.value)
+                            .filter((idxMeta) -> LIFECYCLE_NAME_SETTING.get(idxMeta.getSettings()).equals(policyToDelete))
+                            .map((idxMeta) -> idxMeta.getIndex().getName())
+                            .collect(Collectors.toList());
+                        if (indicesUsingPolicy.isEmpty() == false) {
+                            throw new IllegalArgumentException("Cannot delete policy [" + request.getPolicyName()
+                                + "]. It is in use by one or more indices: " + indicesUsingPolicy);
                         }
                         ClusterState.Builder newState = ClusterState.builder(currentState);
                         IndexLifecycleMetadata currentMetadata = currentState.metaData().custom(IndexLifecycleMetadata.TYPE);