Allow the removal of an in use template if there are other matching ones (#96286)

A data stream can match multiple composable index templates which are
prioritised and the highest priority template is the currently in-use
template. A data stream should always have a matching template, for this
reason when we are removing a composable template we check that it is
not in use. If it is in use, we throw an error and the removal fails.

However, since a data stream can match multiple templates, removing the
currently in use template could mean that the data stream would start
using the next inline. In this case, the removal should be allowed. 

This is what this PR addresses. When we try to remove a composable
template, we do not check if this template is in use, but if this
template is the only template that a data stream could use. Only then we
fail the removal, otherwise the removal succeeds and the data stream
starts using the next in-line template.

Fixes: #96022

docs/changelog/96286.yaml

@@ -0,0 +1,6 @@
+pr: 96286
+summary: Allow the removal of an in-use template if there are other ones matching
+  the dependent data streams
+area: Data streams
+type: bug
+issues: []

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java

@@ -934,7 +934,7 @@ public class MetadataIndexTemplateService {
                 }
             }
             if (templateNames.isEmpty()) {
-                // if its a match all pattern, and no templates are found (we have none), don't
+                // if it's a match all pattern, and no templates are found (we have none), don't
                 // fail with index missing...
                 boolean isMatchAll = false;
                 if (Regex.isMatchAllPattern(name)) {
@@ -948,7 +948,7 @@ public class MetadataIndexTemplateService {
             }
         }
 
-        Set<String> dataStreamsUsingTemplates = dataStreamsUsingTemplates(currentState, templateNames);
+        Set<String> dataStreamsUsingTemplates = dataStreamsExclusivelyUsingTemplates(currentState, templateNames);
         if (dataStreamsUsingTemplates.size() > 0) {
             throw new IllegalArgumentException(
                 "unable to remove composable templates "
@@ -966,7 +966,12 @@ public class MetadataIndexTemplateService {
         return ClusterState.builder(currentState).metadata(metadata).build();
     }
 
-    static Set<String> dataStreamsUsingTemplates(final ClusterState state, final Set<String> templateNames) {
+    /**
+     * Returns the data stream names that solely match the patterns of the template names that were provided and no
+     * other templates. This means that the returned data streams depend on these templates which has implications for
+     * these templates, for example they cannot be removed.
+     */
+    static Set<String> dataStreamsExclusivelyUsingTemplates(final ClusterState state, final Set<String> templateNames) {
         Metadata metadata = state.metadata();
 
         Set<String> namePatterns = templateNames.stream()
@@ -983,10 +988,22 @@ public class MetadataIndexTemplateService {
             // Limit to checking data streams that match any of the templates' index patterns
             .filter(ds -> namePatterns.stream().anyMatch(pattern -> Regex.simpleMatch(pattern, ds.getName())))
             .filter(ds -> {
-                // Retrieve the template that matches the data stream name that has the highest priority
-                String matchedTemplate = findV2Template(metadata, ds.getName(), ds.isHidden());
-                // Limit data streams where their in-use template is the one of specified templates
-                return templateNames.contains(matchedTemplate);
+                // Retrieve the templates that match the data stream name ordered by priority
+                List<Tuple<String, ComposableIndexTemplate>> candidates = findV2CandidateTemplates(metadata, ds.getName(), ds.isHidden());
+                if (candidates.isEmpty()) {
+                    throw new IllegalStateException("Data stream " + ds.getName() + " did not match any composable index templates.");
+                }
+
+                // Limit data streams that can ONLY use any of the specified templates, we do this by filtering
+                // the matching templates that are others than the ones requested and could be a valid template to use.
+                return candidates.stream()
+                    .filter(
+                        template -> templateNames.contains(template.v1()) == false
+                            && isGlobalAndHasIndexHiddenSetting(metadata, template.v2(), template.v1()) == false
+                    )
+                    .map(Tuple::v1)
+                    .toList()
+                    .isEmpty();
             })
             .map(DataStream::getName)
             .collect(Collectors.toSet());
@@ -1183,54 +1200,67 @@ public class MetadataIndexTemplateService {
      */
     @Nullable
     public static String findV2Template(Metadata metadata, String indexName, boolean isHidden) {
+        final List<Tuple<String, ComposableIndexTemplate>> candidates = findV2CandidateTemplates(metadata, indexName, isHidden);
+        if (candidates.isEmpty()) {
+            return null;
+        }
+
+        ComposableIndexTemplate winner = candidates.get(0).v2();
+        String winnerName = candidates.get(0).v1();
+
+        // if the winner template is a global template that specifies the `index.hidden` setting (which is not allowed, so it'd be due to
+        // a restored index cluster state that modified a component template used by this global template such that it has this setting)
+        // we will fail and the user will have to update the index template and remove this setting or update the corresponding component
+        // template that contributes to the index template resolved settings
+        if (isGlobalAndHasIndexHiddenSetting(metadata, winner, winnerName)) {
+            throw new IllegalStateException(
+                "global index template ["
+                    + winnerName
+                    + "], composed of component templates ["
+                    + String.join(",", winner.composedOf())
+                    + "] defined the index.hidden setting, which is not allowed"
+            );
+        }
+
+        return winnerName;
+    }
+
+    /**
+     * Return an ordered list of the name (id) and composable index templates that would apply to an index. The first
+     * one is the winner template that is applied to this index. In the event that no templates are matched,
+     * an empty list is returned.
+     */
+    static List<Tuple<String, ComposableIndexTemplate>> findV2CandidateTemplates(Metadata metadata, String indexName, boolean isHidden) {
         final String resolvedIndexName = IndexNameExpressionResolver.DateMathExpressionResolver.resolveExpression(indexName);
         final Predicate<String> patternMatchPredicate = pattern -> Regex.simpleMatch(pattern, resolvedIndexName);
-        final Map<ComposableIndexTemplate, String> matchedTemplates = new HashMap<>();
+        final List<Tuple<String, ComposableIndexTemplate>> candidates = new ArrayList<>();
         for (Map.Entry<String, ComposableIndexTemplate> entry : metadata.templatesV2().entrySet()) {
             final String name = entry.getKey();
             final ComposableIndexTemplate template = entry.getValue();
             if (isHidden == false) {
                 final boolean matched = template.indexPatterns().stream().anyMatch(patternMatchPredicate);
                 if (matched) {
-                    matchedTemplates.put(template, name);
+                    candidates.add(Tuple.tuple(name, template));
                 }
             } else {
                 final boolean isNotMatchAllTemplate = template.indexPatterns().stream().noneMatch(Regex::isMatchAllPattern);
                 if (isNotMatchAllTemplate) {
                     if (template.indexPatterns().stream().anyMatch(patternMatchPredicate)) {
-                        matchedTemplates.put(template, name);
+                        candidates.add(Tuple.tuple(name, template));
                     }
                 }
             }
         }
 
-        if (matchedTemplates.size() == 0) {
-            return null;
-        }
-
-        final List<ComposableIndexTemplate> candidates = new ArrayList<>(matchedTemplates.keySet());
-        CollectionUtil.timSort(candidates, Comparator.comparing(ComposableIndexTemplate::priorityOrZero, Comparator.reverseOrder()));
-
-        assert candidates.size() > 0 : "we should have returned early with no candidates";
-        ComposableIndexTemplate winner = candidates.get(0);
-        String winnerName = matchedTemplates.get(winner);
-
-        // if the winner template is a global template that specifies the `index.hidden` setting (which is not allowed, so it'd be due to
-        // a restored index cluster state that modified a component template used by this global template such that it has this setting)
-        // we will fail and the user will have to update the index template and remove this setting or update the corresponding component
-        // template that contributes to the index template resolved settings
-        if (winner.indexPatterns().stream().anyMatch(Regex::isMatchAllPattern)
-            && IndexMetadata.INDEX_HIDDEN_SETTING.exists(resolveSettings(metadata, winnerName))) {
-            throw new IllegalStateException(
-                "global index template ["
-                    + winnerName
-                    + "], composed of component templates ["
-                    + String.join(",", winner.composedOf())
-                    + "] defined the index.hidden setting, which is not allowed"
-            );
-        }
+        CollectionUtil.timSort(candidates, Comparator.comparing(candidate -> candidate.v2().priorityOrZero(), Comparator.reverseOrder()));
+        return candidates;
+    }
 
-        return winnerName;
+    // Checks if a global template specifies the `index.hidden` setting. This check is important because a global
+    // template shouldn't specify the `index.hidden` setting, we leave it up to the caller to handle this situation.
+    private static boolean isGlobalAndHasIndexHiddenSetting(Metadata metadata, ComposableIndexTemplate template, String templateName) {
+        return template.indexPatterns().stream().anyMatch(Regex::isMatchAllPattern)
+            && IndexMetadata.INDEX_HIDDEN_SETTING.exists(resolveSettings(metadata, templateName));
     }
 
     /**

server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java

@@ -2140,13 +2140,69 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
             containsString("unable to remove composable templates [logs, logs2] as they are in use by a data streams [logs-mysql-default]")
         );
 
-        assertThat(MetadataIndexTemplateService.dataStreamsUsingTemplates(state, Set.of("logs")), equalTo(Set.of()));
-        assertThat(MetadataIndexTemplateService.dataStreamsUsingTemplates(state, Set.of("logs2")), equalTo(Set.of("logs-mysql-default")));
+        assertThat(MetadataIndexTemplateService.dataStreamsExclusivelyUsingTemplates(state, Set.of("logs")), equalTo(Set.of()));
+        assertThat(MetadataIndexTemplateService.findV2Template(state.metadata(), "logs-mysql-default", false), equalTo("logs2"));
 
         // The unreferenced template can be removed without an exception
         MetadataIndexTemplateService.innerRemoveIndexTemplateV2(stateWithTwoTemplates, "logs");
     }
 
+    public void testRemovingHigherOrderTemplateOfDataStreamWithMultipleTemplates() throws Exception {
+        ClusterState state = ClusterState.EMPTY_STATE;
+        final MetadataIndexTemplateService service = getMetadataIndexTemplateService();
+
+        ComposableIndexTemplate template = new ComposableIndexTemplate(
+            Collections.singletonList("logs-*"),
+            null,
+            null,
+            100L,
+            null,
+            null,
+            new ComposableIndexTemplate.DataStreamTemplate(),
+            null
+        );
+
+        state = service.addIndexTemplateV2(state, false, "logs", template);
+
+        ClusterState stateWithDS = ClusterState.builder(state)
+            .metadata(
+                Metadata.builder(state.metadata())
+                    .put(
+                        DataStreamTestHelper.newInstance(
+                            "logs-mysql-default",
+                            Collections.singletonList(new Index(".ds-logs-mysql-default-000001", "uuid"))
+                        )
+                    )
+                    .put(
+                        IndexMetadata.builder(".ds-logs-mysql-default-000001")
+                            .settings(indexSettings(Version.CURRENT, 1, 0).put(IndexMetadata.SETTING_INDEX_UUID, "uuid"))
+                    )
+            )
+            .build();
+
+        ComposableIndexTemplate fineGrainedLogsTemplate = new ComposableIndexTemplate(
+            Collections.singletonList("logs-mysql-*"),
+            null,
+            null,
+            200L, // Higher priority
+            null,
+            null,
+            new ComposableIndexTemplate.DataStreamTemplate(),
+            null
+        );
+
+        state = service.addIndexTemplateV2(stateWithDS, false, "logs-test", fineGrainedLogsTemplate);
+
+        // Verify that the data stream now matches to the higher order template
+        assertThat(MetadataIndexTemplateService.dataStreamsExclusivelyUsingTemplates(state, Set.of("logs")), equalTo(Set.of()));
+        assertThat(MetadataIndexTemplateService.findV2Template(state.metadata(), "logs-mysql-default", false), equalTo("logs-test"));
+
+        // Test removing the higher order template
+        state = MetadataIndexTemplateService.innerRemoveIndexTemplateV2(state, "logs-test");
+
+        assertThat(MetadataIndexTemplateService.findV2Template(state.metadata(), "logs-mysql-default", false), equalTo("logs"));
+    }
+
     public void testV2TemplateOverlaps() throws Exception {
         {
             ComposableIndexTemplate template = new ComposableIndexTemplate(