Optimize usage calculation in ILM policies retrieval API (#106953)

Optimize calculating the usage of ILM policies in the `GET _ilm/policy` and `GET _ilm/policy/<policy_id>` endpoints by xtracting a separate class that pre-computes some parts on initialization (i.e. only once per request) and then uses those pre-computed parts when calculating the usage for an individual policy. By precomputing all the usages, the class makes a tradeoff by using a little bit more memory to significantly improve the overall processing time.

docs/changelog/106953.yaml

@@ -0,0 +1,6 @@
+pr: 106953
+summary: Optimize usage calculation in ILM policies retrieval API
+area: ILM+SLM
+type: enhancement
+issues:
+ - 105773

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java

@@ -62,6 +62,7 @@ import java.io.IOException;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
@@ -1087,9 +1088,8 @@ public class MetadataIndexTemplateService {
             .map(templateName -> projectMetadata.templatesV2().get(templateName))
             .filter(Objects::nonNull)
             .map(ComposableIndexTemplate::indexPatterns)
-            .map(Set::copyOf)
-            .reduce(Sets::union)
-            .orElse(Set.of());
+            .flatMap(List::stream)
+            .collect(Collectors.toSet());
 
         return projectMetadata.dataStreams()
             .values()
@@ -1099,9 +1099,10 @@ public class MetadataIndexTemplateService {
             .filter(ds -> {
                 // Retrieve the templates that match the data stream name ordered by priority
                 List<Tuple<String, ComposableIndexTemplate>> candidates = findV2CandidateTemplates(
-                    projectMetadata,
+                    projectMetadata.templatesV2().entrySet(),
                     ds.getName(),
-                    ds.isHidden()
+                    ds.isHidden(),
+                    false
                 );
                 if (candidates.isEmpty()) {
                     throw new IllegalStateException("Data stream " + ds.getName() + " did not match any composable index templates.");
@@ -1110,13 +1111,10 @@ public class MetadataIndexTemplateService {
                 // Limit data streams that can ONLY use any of the specified templates, we do this by filtering
                 // the matching templates that are others than the ones requested and could be a valid template to use.
                 return candidates.stream()
-                    .filter(
+                    .noneMatch(
                         template -> templateNames.contains(template.v1()) == false
                             && isGlobalAndHasIndexHiddenSetting(projectMetadata, template.v2(), template.v1()) == false
-                    )
-                    .map(Tuple::v1)
-                    .toList()
-                    .isEmpty();
+                    );
             })
             .map(DataStream::getName)
             .collect(Collectors.toSet());
@@ -1357,7 +1355,41 @@ public class MetadataIndexTemplateService {
      */
     @Nullable
     public static String findV2Template(ProjectMetadata projectMetadata, String indexName, boolean isHidden) {
-        final List<Tuple<String, ComposableIndexTemplate>> candidates = findV2CandidateTemplates(projectMetadata, indexName, isHidden);
+        return findV2Template(projectMetadata, projectMetadata.templatesV2().entrySet(), indexName, isHidden, false);
+    }
+
+    /**
+     * Return the name (id) of the highest matching index template out of the provided templates (that <i>need</i> to be sorted descending
+     * on priority beforehand), or the given index name. In the event that no templates are matched, {@code null} is returned.
+     */
+    @Nullable
+    public static String findV2TemplateFromSortedList(
+        ProjectMetadata projectMetadata,
+        Collection<Map.Entry<String, ComposableIndexTemplate>> templates,
+        String indexName,
+        boolean isHidden
+    ) {
+        return findV2Template(projectMetadata, templates, indexName, isHidden, true);
+    }
+
+    /**
+     * Return the name (id) of the highest matching index template, out of the provided templates, for the given index name. In
+     * the event that no templates are matched, {@code null} is returned.
+     */
+    @Nullable
+    private static String findV2Template(
+        ProjectMetadata projectMetadata,
+        Collection<Map.Entry<String, ComposableIndexTemplate>> templates,
+        String indexName,
+        boolean isHidden,
+        boolean exitOnFirstMatch
+    ) {
+        final List<Tuple<String, ComposableIndexTemplate>> candidates = findV2CandidateTemplates(
+            templates,
+            indexName,
+            isHidden,
+            exitOnFirstMatch
+        );
         if (candidates.isEmpty()) {
             return null;
         }
@@ -1386,16 +1418,25 @@ public class MetadataIndexTemplateService {
      * Return an ordered list of the name (id) and composable index templates that would apply to an index. The first
      * one is the winner template that is applied to this index. In the event that no templates are matched,
      * an empty list is returned.
+     * @param templates a list of template entries (name, template) - needs to be sorted when {@code exitOnFirstMatch} is {@code true}
+     * @param indexName the index (or data stream) name that should be used for matching the index patterns on the templates
+     * @param isHidden whether {@code indexName} belongs to a hidden index - this option is redundant for data streams, as backing indices
+     *                 of data streams will always be returned, regardless of whether the data stream is hidden or not
+     * @param exitOnFirstMatch if true, we return immediately after finding a match. That means that the <code>templates</code>
+     *                         parameter needs to be sorted based on priority (descending) for this method to return a sensible result,
+     *                         otherwise this method would just return the first template that matches the name, in an unspecified order
      */
-    static List<Tuple<String, ComposableIndexTemplate>> findV2CandidateTemplates(
-        ProjectMetadata projectMetadata,
+    private static List<Tuple<String, ComposableIndexTemplate>> findV2CandidateTemplates(
+        Collection<Map.Entry<String, ComposableIndexTemplate>> templates,
         String indexName,
-        boolean isHidden
+        boolean isHidden,
+        boolean exitOnFirstMatch
     ) {
+        assert exitOnFirstMatch == false || areTemplatesSorted(templates) : "Expected templates to be sorted";
         final String resolvedIndexName = IndexNameExpressionResolver.DateMathExpressionResolver.resolveExpression(indexName);
         final Predicate<String> patternMatchPredicate = pattern -> Regex.simpleMatch(pattern, resolvedIndexName);
         final List<Tuple<String, ComposableIndexTemplate>> candidates = new ArrayList<>();
-        for (Map.Entry<String, ComposableIndexTemplate> entry : projectMetadata.templatesV2().entrySet()) {
+        for (Map.Entry<String, ComposableIndexTemplate> entry : templates) {
             final String name = entry.getKey();
             final ComposableIndexTemplate template = entry.getValue();
             /*
@@ -1403,16 +1444,13 @@ public class MetadataIndexTemplateService {
              * and we do want to return even match-all templates for those. Not doing so can result in a situation where a data stream is
              * built with a template that none of its indices match.
              */
-            if (isHidden == false || template.getDataStreamTemplate() != null) {
-                if (anyMatch(template.indexPatterns(), patternMatchPredicate)) {
-                    candidates.add(Tuple.tuple(name, template));
-                }
-            } else {
-                final boolean isNotMatchAllTemplate = noneMatch(template.indexPatterns(), Regex::isMatchAllPattern);
-                if (isNotMatchAllTemplate) {
-                    if (anyMatch(template.indexPatterns(), patternMatchPredicate)) {
-                        candidates.add(Tuple.tuple(name, template));
-                    }
+            if (anyMatch(template.indexPatterns(), Regex::isMatchAllPattern) && isHidden && template.getDataStreamTemplate() == null) {
+                continue;
+            }
+            if (anyMatch(template.indexPatterns(), patternMatchPredicate)) {
+                candidates.add(Tuple.tuple(name, template));
+                if (exitOnFirstMatch) {
+                    return candidates;
                 }
             }
         }
@@ -1421,6 +1459,17 @@ public class MetadataIndexTemplateService {
         return candidates;
     }
 
+    private static boolean areTemplatesSorted(Collection<Map.Entry<String, ComposableIndexTemplate>> templates) {
+        ComposableIndexTemplate previousTemplate = null;
+        for (Map.Entry<String, ComposableIndexTemplate> template : templates) {
+            if (previousTemplate != null && template.getValue().priorityOrZero() > previousTemplate.priorityOrZero()) {
+                return false;
+            }
+            previousTemplate = template.getValue();
+        }
+        return true;
+    }
+
     // Checks if a global template specifies the `index.hidden` setting. This check is important because a global
     // template shouldn't specify the `index.hidden` setting, we leave it up to the caller to handle this situation.
     private static boolean isGlobalAndHasIndexHiddenSetting(

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculator.java

@@ -0,0 +1,125 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.ilm;
+
+import org.apache.lucene.util.CollectionUtil;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.ItemUsage;
+import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
+import org.elasticsearch.cluster.metadata.ProjectMetadata;
+import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.Maps;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A class that can be used to calculate the usages of ILM policies across the cluster. By precomputing all the usages,
+ * the class makes a tradeoff by using a little bit more memory to significantly improve the overall processing time.
+ */
+public class LifecyclePolicyUsageCalculator {
+
+    /** A map from policy name to list of composable templates that use that policy. */
+    private final Map<String, List<String>> policyToTemplates;
+    /** A map from policy name to list of data streams that use that policy. */
+    private final Map<String, List<String>> policyToDataStreams;
+    /** A map from policy name to list of indices that use that policy. */
+    private final Map<String, List<String>> policyToIndices;
+
+    public LifecyclePolicyUsageCalculator(
+        final IndexNameExpressionResolver indexNameExpressionResolver,
+        ProjectMetadata project,
+        List<String> requestedPolicyNames
+    ) {
+        final IndexLifecycleMetadata ilmMetadata = project.custom(IndexLifecycleMetadata.TYPE);
+        // We're making a bet here that if the `name` contains a wildcard, there's a large chance it'll simply match all policies.
+        final var expectedSize = Regex.isSimpleMatchPattern(requestedPolicyNames.get(0))
+            ? ilmMetadata.getPolicyMetadatas().size()
+            : requestedPolicyNames.size();
+
+        // We keep a map from composable template name to policy name to avoid having to resolve the template settings to determine
+        // the template's policy twice.
+        final Map<String, String> templateToPolicy = new HashMap<>();
+
+        // Build the map of which policy is used by which index templates.
+        policyToTemplates = Maps.newHashMapWithExpectedSize(expectedSize);
+        for (Map.Entry<String, ComposableIndexTemplate> entry : project.templatesV2().entrySet()) {
+            Settings settings = MetadataIndexTemplateService.resolveSettings(entry.getValue(), project.componentTemplates());
+            final var policyName = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(settings);
+            // We only store the template if its policy matched any of the requested names.
+            if (doesPolicyMatchAnyName(policyName, requestedPolicyNames) == false) {
+                continue;
+            }
+            policyToTemplates.computeIfAbsent(policyName, k -> new ArrayList<>()).add(entry.getKey());
+            templateToPolicy.put(entry.getKey(), policyName);
+        }
+
+        // Sort all templates by descending priority. That way, findV2Template can exit on the first-matched template.
+        final var indexTemplates = new ArrayList<>(project.templatesV2().entrySet());
+        CollectionUtil.timSort(indexTemplates, Comparator.comparing(entry -> entry.getValue().priorityOrZero(), Comparator.reverseOrder()));
+
+        // Build the map of which policy is used by which data streams.
+        policyToDataStreams = Maps.newHashMapWithExpectedSize(expectedSize);
+        final List<String> allDataStreams = indexNameExpressionResolver.dataStreamNames(
+            project,
+            IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN_NO_SELECTOR
+        );
+        for (String dataStream : allDataStreams) {
+            // Find the index template with the highest priority that matches this data stream's name.
+            String indexTemplate = MetadataIndexTemplateService.findV2TemplateFromSortedList(project, indexTemplates, dataStream, false);
+            if (indexTemplate == null) {
+                assert false : "Data stream [" + dataStream + "] has no matching template";
+                continue;
+            }
+            final var policyName = templateToPolicy.get(indexTemplate);
+            // If there was no entry, either the template didn't specify an ILM policy or the policy didn't match any of the requested names
+            if (policyName == null) {
+                continue;
+            }
+            policyToDataStreams.computeIfAbsent(policyName, k -> new ArrayList<>()).add(dataStream);
+        }
+
+        // Build the map of which policy is used by which indices.
+        policyToIndices = Maps.newHashMapWithExpectedSize(expectedSize);
+        for (IndexMetadata indexMetadata : project.indices().values()) {
+            final var policyName = indexMetadata.getLifecyclePolicyName();
+            // We only store the index if its policy matched any of the specified names.
+            if (doesPolicyMatchAnyName(policyName, requestedPolicyNames) == false) {
+                continue;
+            }
+            policyToIndices.computeIfAbsent(policyName, k -> new ArrayList<>()).add(indexMetadata.getIndex().getName());
+        }
+    }
+
+    /**
+     * Retrieves the pre-calculated indices, data streams, and composable templates that use the given policy.
+     */
+    public ItemUsage retrieveCalculatedUsage(String policyName) {
+        return new ItemUsage(
+            policyToIndices.getOrDefault(policyName, List.of()),
+            policyToDataStreams.getOrDefault(policyName, List.of()),
+            policyToTemplates.getOrDefault(policyName, List.of())
+        );
+    }
+
+    private boolean doesPolicyMatchAnyName(String policyName, List<String> names) {
+        for (var name : names) {
+            if (Regex.simpleMatch(name, policyName)) {
+                return true;
+            }
+        }
+        return false;
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java

@@ -8,14 +8,8 @@
 package org.elasticsearch.xpack.core.ilm;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.metadata.ItemUsage;
-import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
-import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.compress.NotXContentException;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.XContentParser;
@@ -24,7 +18,6 @@ import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.core.template.resources.TemplateResources;
 
 import java.io.IOException;
-import java.util.List;
 import java.util.Map;
 
 /**
@@ -104,43 +97,4 @@ public class LifecyclePolicyUtils {
             throw new ElasticsearchParseException("invalid policy", e);
         }
     }
-
-    /**
-     * Given a cluster state and ILM policy, calculate the {@link ItemUsage} of
-     * the policy (what indices, data streams, and templates use the policy)
-     */
-    public static ItemUsage calculateUsage(
-        final IndexNameExpressionResolver indexNameExpressionResolver,
-        final ProjectMetadata project,
-        final String policyName
-    ) {
-        final List<String> indices = project.indices()
-            .values()
-            .stream()
-            .filter(indexMetadata -> policyName.equals(indexMetadata.getLifecyclePolicyName()))
-            .map(indexMetadata -> indexMetadata.getIndex().getName())
-            .toList();
-
-        final List<String> allDataStreams = indexNameExpressionResolver.dataStreamNames(
-            project,
-            IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN_NO_SELECTOR
-        );
-
-        final List<String> dataStreams = allDataStreams.stream().filter(dsName -> {
-            String indexTemplate = MetadataIndexTemplateService.findV2Template(project, dsName, false);
-            if (indexTemplate != null) {
-                Settings settings = MetadataIndexTemplateService.resolveSettings(project, indexTemplate);
-                return policyName.equals(LifecycleSettings.LIFECYCLE_NAME_SETTING.get(settings));
-            } else {
-                return false;
-            }
-        }).toList();
-
-        final List<String> composableTemplates = project.templatesV2().keySet().stream().filter(templateName -> {
-            Settings settings = MetadataIndexTemplateService.resolveSettings(project, templateName);
-            return policyName.equals(LifecycleSettings.LIFECYCLE_NAME_SETTING.get(settings));
-        }).toList();
-
-        return new ItemUsage(indices, dataStreams, composableTemplates);
-    }
 }

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculatorTests.java

@@ -0,0 +1,221 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.ilm;
+
+import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
+import org.elasticsearch.cluster.metadata.ComposableIndexTemplateMetadata;
+import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.ItemUsage;
+import org.elasticsearch.cluster.metadata.ProjectMetadata;
+import org.elasticsearch.cluster.metadata.Template;
+import org.elasticsearch.cluster.project.TestProjectResolvers;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.indices.EmptySystemIndices;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
+
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class LifecyclePolicyUsageCalculatorTests extends ESTestCase {
+
+    private IndexNameExpressionResolver iner;
+
+    @Before
+    public void init() {
+        iner = new IndexNameExpressionResolver(
+            new ThreadContext(Settings.EMPTY),
+            EmptySystemIndices.INSTANCE,
+            TestProjectResolvers.alwaysThrow()
+        );
+    }
+
+    public void testGetUsageNonExistentPolicy() {
+        // Test where policy does not exist
+        final var project = ProjectMetadata.builder(randomProjectIdOrDefault()).build();
+        assertThat(
+            new LifecyclePolicyUsageCalculator(iner, project, List.of("mypolicy")).retrieveCalculatedUsage("mypolicy"),
+            equalTo(new ItemUsage(List.of(), List.of(), List.of()))
+        );
+    }
+
+    public void testGetUsageUnusedPolicy() {
+        // Test where policy is not used by anything
+        final var project = ProjectMetadata.builder(randomProjectIdOrDefault())
+            .putCustom(
+                IndexLifecycleMetadata.TYPE,
+                new IndexLifecycleMetadata(
+                    Map.of("mypolicy", LifecyclePolicyMetadataTests.createRandomPolicyMetadata("mypolicy")),
+                    OperationMode.RUNNING
+                )
+            )
+            .build();
+
+        assertThat(
+            new LifecyclePolicyUsageCalculator(iner, project, List.of("mypolicy")).retrieveCalculatedUsage("mypolicy"),
+            equalTo(new ItemUsage(List.of(), List.of(), List.of()))
+        );
+    }
+
+    public void testGetUsagePolicyUsedByIndex() {
+        // Test where policy exists and is used by an index
+        final var project = ProjectMetadata.builder(randomProjectIdOrDefault())
+            .putCustom(
+                IndexLifecycleMetadata.TYPE,
+                new IndexLifecycleMetadata(
+                    Map.of("mypolicy", LifecyclePolicyMetadataTests.createRandomPolicyMetadata("mypolicy")),
+                    OperationMode.RUNNING
+                )
+            )
+            .put(
+                IndexMetadata.builder("myindex")
+                    .settings(indexSettings(IndexVersion.current(), 1, 0).put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy"))
+            )
+            .build();
+
+        assertThat(
+            new LifecyclePolicyUsageCalculator(iner, project, List.of("mypolicy")).retrieveCalculatedUsage("mypolicy"),
+            equalTo(new ItemUsage(List.of("myindex"), List.of(), List.of()))
+        );
+    }
+
+    public void testGetUsagePolicyUsedByIndexAndTemplate() {
+        // Test where policy exists and is used by an index, and template
+        final var project = ProjectMetadata.builder(randomProjectIdOrDefault())
+            .putCustom(
+                IndexLifecycleMetadata.TYPE,
+                new IndexLifecycleMetadata(
+                    Map.of("mypolicy", LifecyclePolicyMetadataTests.createRandomPolicyMetadata("mypolicy")),
+                    OperationMode.RUNNING
+                )
+            )
+            .put(
+                IndexMetadata.builder("myindex")
+                    .settings(indexSettings(IndexVersion.current(), 1, 0).put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy"))
+            )
+            .putCustom(
+                ComposableIndexTemplateMetadata.TYPE,
+                new ComposableIndexTemplateMetadata(
+                    Map.of(
+                        "mytemplate",
+                        ComposableIndexTemplate.builder()
+                            .indexPatterns(List.of("myds"))
+                            .template(
+                                new Template(Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy").build(), null, null)
+                            )
+                            .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false))
+                            .build()
+                    )
+                )
+            )
+            .build();
+
+        assertThat(
+            new LifecyclePolicyUsageCalculator(iner, project, List.of("mypolicy")).retrieveCalculatedUsage("mypolicy"),
+            equalTo(new ItemUsage(List.of("myindex"), List.of(), List.of("mytemplate")))
+        );
+    }
+
+    public void testGetUsagePolicyUsedByIndexAndTemplateAndDataStream() {
+        // Test where policy exists and is used by an index, data stream, and template
+        IndexMetadata index = IndexMetadata.builder("myindex")
+            .settings(indexSettings(IndexVersion.current(), 1, 0).put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy"))
+            .build();
+        final var project = ProjectMetadata.builder(randomProjectIdOrDefault())
+            .putCustom(
+                IndexLifecycleMetadata.TYPE,
+                new IndexLifecycleMetadata(
+                    Map.of("mypolicy", LifecyclePolicyMetadataTests.createRandomPolicyMetadata("mypolicy")),
+                    OperationMode.RUNNING
+                )
+            )
+            .put(index, false)
+            .put(DataStreamTestHelper.newInstance("myds", List.of(index.getIndex())))
+            .put(
+                IndexMetadata.builder("another")
+                    .settings(indexSettings(IndexVersion.current(), 1, 0).put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy"))
+            )
+            .put(
+                IndexMetadata.builder("other")
+                    .settings(indexSettings(IndexVersion.current(), 1, 0).put(LifecycleSettings.LIFECYCLE_NAME, "otherpolicy"))
+            )
+
+            .putCustom(
+                ComposableIndexTemplateMetadata.TYPE,
+                new ComposableIndexTemplateMetadata(
+                    Map.of(
+                        "mytemplate",
+                        ComposableIndexTemplate.builder()
+                            .indexPatterns(List.of("myds"))
+                            .template(
+                                new Template(Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy").build(), null, null)
+                            )
+                            .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false))
+                            .build()
+                    )
+                )
+            )
+            .build();
+
+        assertThat(
+            new LifecyclePolicyUsageCalculator(iner, project, List.of("mypolicy")).retrieveCalculatedUsage("mypolicy"),
+            equalTo(new ItemUsage(List.of("myindex", "another"), List.of("myds"), List.of("mytemplate")))
+        );
+    }
+
+    public void testGetUsagePolicyNotUsedByDataStreamDueToOverride() {
+        // Test when a data stream does not use the policy anymore because of a higher template
+        IndexMetadata index = IndexMetadata.builder("myindex")
+            .settings(indexSettings(IndexVersion.current(), 1, 0).put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy"))
+            .build();
+        final var project = ProjectMetadata.builder(randomProjectIdOrDefault())
+            .put(index, false)
+            .put(DataStreamTestHelper.newInstance("myds", List.of(index.getIndex())))
+            .putCustom(
+                IndexLifecycleMetadata.TYPE,
+                new IndexLifecycleMetadata(
+                    Map.of("mypolicy", LifecyclePolicyMetadataTests.createRandomPolicyMetadata("mypolicy")),
+                    OperationMode.RUNNING
+                )
+            )
+            .putCustom(
+                ComposableIndexTemplateMetadata.TYPE,
+                new ComposableIndexTemplateMetadata(
+                    Map.of(
+                        "mytemplate",
+                        ComposableIndexTemplate.builder()
+                            .indexPatterns(List.of("myds*"))
+                            .template(
+                                new Template(Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy").build(), null, null)
+                            )
+                            .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false))
+                            .build(),
+                        "myhighertemplate",
+                        ComposableIndexTemplate.builder()
+                            .indexPatterns(List.of("myds"))
+                            .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false))
+                            .priority(1_000L)
+                            .build()
+                    )
+                )
+            )
+            .build();
+
+        // Test where policy exists and is used by an index, datastream, and template
+        assertThat(
+            new LifecyclePolicyUsageCalculator(iner, project, List.of("mypolicy")).retrieveCalculatedUsage("mypolicy"),
+            equalTo(new ItemUsage(List.of("myindex"), List.of(), List.of("mytemplate")))
+        );
+    }
+}

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtilsTests.java

@@ -1,162 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.core.ilm;
-
-import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
-import org.elasticsearch.cluster.metadata.ComposableIndexTemplateMetadata;
-import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
-import org.elasticsearch.cluster.metadata.IndexMetadata;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.metadata.ItemUsage;
-import org.elasticsearch.cluster.metadata.ProjectMetadata;
-import org.elasticsearch.cluster.metadata.Template;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.IndexVersion;
-import org.elasticsearch.indices.TestIndexNameExpressionResolver;
-import org.elasticsearch.test.ESTestCase;
-
-import java.util.List;
-import java.util.Map;
-
-import static org.hamcrest.Matchers.equalTo;
-
-public class LifecyclePolicyUtilsTests extends ESTestCase {
-    public void testCalculateUsage() {
-        final IndexNameExpressionResolver iner = TestIndexNameExpressionResolver.newInstance();
-
-        {
-            // Test where policy does not exist
-            var project = ProjectMetadata.builder(randomProjectIdOrDefault()).build();
-            assertThat(
-                LifecyclePolicyUtils.calculateUsage(iner, project, "mypolicy"),
-                equalTo(new ItemUsage(List.of(), List.of(), List.of()))
-            );
-        }
-
-        {
-            // Test where policy is not used by anything
-            var project = ProjectMetadata.builder(randomProjectIdOrDefault())
-                .putCustom(
-                    IndexLifecycleMetadata.TYPE,
-                    new IndexLifecycleMetadata(
-                        Map.of("mypolicy", LifecyclePolicyMetadataTests.createRandomPolicyMetadata("mypolicy")),
-                        OperationMode.RUNNING
-                    )
-                )
-                .build();
-            assertThat(
-                LifecyclePolicyUtils.calculateUsage(iner, project, "mypolicy"),
-                equalTo(new ItemUsage(List.of(), List.of(), List.of()))
-            );
-        }
-
-        {
-            // Test where policy exists and is used by an index
-            var project = ProjectMetadata.builder(randomProjectIdOrDefault())
-                .putCustom(
-                    IndexLifecycleMetadata.TYPE,
-                    new IndexLifecycleMetadata(
-                        Map.of("mypolicy", LifecyclePolicyMetadataTests.createRandomPolicyMetadata("mypolicy")),
-                        OperationMode.RUNNING
-                    )
-                )
-                .put(
-                    IndexMetadata.builder("myindex")
-                        .settings(indexSettings(IndexVersion.current(), 1, 0).put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy"))
-                )
-                .build();
-            assertThat(
-                LifecyclePolicyUtils.calculateUsage(iner, project, "mypolicy"),
-                equalTo(new ItemUsage(List.of("myindex"), List.of(), List.of()))
-            );
-        }
-
-        {
-            // Test where policy exists and is used by an index, and template
-            var project = ProjectMetadata.builder(randomProjectIdOrDefault())
-                .putCustom(
-                    IndexLifecycleMetadata.TYPE,
-                    new IndexLifecycleMetadata(
-                        Map.of("mypolicy", LifecyclePolicyMetadataTests.createRandomPolicyMetadata("mypolicy")),
-                        OperationMode.RUNNING
-                    )
-                )
-                .put(
-                    IndexMetadata.builder("myindex")
-                        .settings(indexSettings(IndexVersion.current(), 1, 0).put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy"))
-                )
-                .putCustom(
-                    ComposableIndexTemplateMetadata.TYPE,
-                    new ComposableIndexTemplateMetadata(
-                        Map.of(
-                            "mytemplate",
-                            ComposableIndexTemplate.builder()
-                                .indexPatterns(List.of("myds"))
-                                .template(
-                                    new Template(Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy").build(), null, null)
-                                )
-                                .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false))
-                                .build()
-                        )
-                    )
-                )
-                .build();
-            assertThat(
-                LifecyclePolicyUtils.calculateUsage(iner, project, "mypolicy"),
-                equalTo(new ItemUsage(List.of("myindex"), List.of(), List.of("mytemplate")))
-            );
-        }
-
-        {
-            var projectBuilder = ProjectMetadata.builder(randomProjectIdOrDefault())
-                .putCustom(
-                    IndexLifecycleMetadata.TYPE,
-                    new IndexLifecycleMetadata(
-                        Map.of("mypolicy", LifecyclePolicyMetadataTests.createRandomPolicyMetadata("mypolicy")),
-                        OperationMode.RUNNING
-                    )
-                )
-                .put(
-                    IndexMetadata.builder("myindex")
-                        .settings(indexSettings(IndexVersion.current(), 1, 0).put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy"))
-                )
-                .put(
-                    IndexMetadata.builder("another")
-                        .settings(indexSettings(IndexVersion.current(), 1, 0).put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy"))
-                )
-                .put(
-                    IndexMetadata.builder("other")
-                        .settings(indexSettings(IndexVersion.current(), 1, 0).put(LifecycleSettings.LIFECYCLE_NAME, "otherpolicy"))
-                )
-
-                .putCustom(
-                    ComposableIndexTemplateMetadata.TYPE,
-                    new ComposableIndexTemplateMetadata(
-                        Map.of(
-                            "mytemplate",
-                            ComposableIndexTemplate.builder()
-                                .indexPatterns(List.of("myds"))
-                                .template(
-                                    new Template(Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy").build(), null, null)
-                                )
-                                .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false))
-                                .build()
-                        )
-                    )
-                );
-            // Need to get the real Index instance of myindex:
-            projectBuilder.put(DataStreamTestHelper.newInstance("myds", List.of(projectBuilder.get("myindex").getIndex())));
-
-            // Test where policy exists and is used by an index, datastream, and template
-            assertThat(
-                LifecyclePolicyUtils.calculateUsage(iner, projectBuilder.build(), "mypolicy"),
-                equalTo(new ItemUsage(List.of("myindex", "another"), List.of("myds"), List.of("mytemplate")))
-            );
-        }
-    }
-}

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportGetLifecycleAction.java

@@ -25,7 +25,7 @@ import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata;
 import org.elasticsearch.xpack.core.ilm.LifecyclePolicyMetadata;
-import org.elasticsearch.xpack.core.ilm.LifecyclePolicyUtils;
+import org.elasticsearch.xpack.core.ilm.LifecyclePolicyUsageCalculator;
 import org.elasticsearch.xpack.core.ilm.action.GetLifecycleAction;
 import org.elasticsearch.xpack.core.ilm.action.GetLifecycleAction.LifecyclePolicyResponseItem;
 import org.elasticsearch.xpack.core.ilm.action.GetLifecycleAction.Request;
@@ -97,6 +97,7 @@ public class TransportGetLifecycleAction extends TransportMasterNodeAction<Reque
                 );
             }
 
+            var lifecyclePolicyUsageCalculator = new LifecyclePolicyUsageCalculator(indexNameExpressionResolver, project, names);
             Map<String, LifecyclePolicyResponseItem> policyResponseItemMap = new LinkedHashMap<>();
             for (String name : names) {
                 if (Regex.isSimpleMatchPattern(name)) {
@@ -112,7 +113,7 @@ public class TransportGetLifecycleAction extends TransportMasterNodeAction<Reque
                                     policyMetadata.getPolicy(),
                                     policyMetadata.getVersion(),
                                     policyMetadata.getModifiedDateString(),
-                                    LifecyclePolicyUtils.calculateUsage(indexNameExpressionResolver, project, policyMetadata.getName())
+                                    lifecyclePolicyUsageCalculator.retrieveCalculatedUsage(policyMetadata.getName())
                                 )
                             );
                         }
@@ -129,7 +130,7 @@ public class TransportGetLifecycleAction extends TransportMasterNodeAction<Reque
                             policyMetadata.getPolicy(),
                             policyMetadata.getVersion(),
                             policyMetadata.getModifiedDateString(),
-                            LifecyclePolicyUtils.calculateUsage(indexNameExpressionResolver, project, policyMetadata.getName())
+                            lifecyclePolicyUsageCalculator.retrieveCalculatedUsage(policyMetadata.getName())
                         )
                     );
                 }