Search all (_all, *) resolves data streams too (#58869)

* Search all (_all, *) to resolves data streams too

* Resolve backing indices when resolving the wildcard

* Remove ArrayList instance

* Crikey fix post-merge compilation error

* Update test comments

* Tests: randomise "include all" wildcard expressions

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

server/src/internalClusterTest/java/org/elasticsearch/indices/DataStreamIT.java

@@ -69,8 +69,8 @@ import java.util.Arrays;
 import java.util.Comparator;
 import java.util.List;
 import java.util.Locale;
-import java.util.Optional;
 import java.util.Map;
+import java.util.Optional;
 
 import static org.elasticsearch.indices.IndicesOptionsIntegrationIT._flush;
 import static org.elasticsearch.indices.IndicesOptionsIntegrationIT.clearCache;
@@ -717,6 +717,32 @@ public class DataStreamIT extends ESIntegTestCase {
         assertThat(response.getIndex(), equalTo(DataStream.getDefaultBackingIndexName("logs-foobar", 1)));
     }
 
+    public void testSearchAllResolvesDataStreams() throws Exception {
+        putComposableIndexTemplate("id1", "@timestamp1", List.of("metrics-foo*"));
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request("metrics-foo");
+        client().admin().indices().createDataStream(createDataStreamRequest).get();
+
+        putComposableIndexTemplate("id2", "@timestamp2", List.of("metrics-bar*"));
+        createDataStreamRequest = new CreateDataStreamAction.Request("metrics-bar");
+        client().admin().indices().createDataStream(createDataStreamRequest).get();
+
+        int numDocsBar = randomIntBetween(2, 16);
+        indexDocs("metrics-bar", "@timestamp2", numDocsBar);
+        int numDocsFoo = randomIntBetween(2, 16);
+        indexDocs("metrics-foo", "@timestamp1", numDocsFoo);
+
+        RolloverResponse rolloverResponse = client().admin().indices().rolloverIndex(new RolloverRequest("metrics-foo", null)).get();
+        assertThat(rolloverResponse.getNewIndex(), equalTo(DataStream.getDefaultBackingIndexName("metrics-foo", 2)));
+
+        // ingest some more data in the rolled data stream
+        int numDocsRolledFoo = randomIntBetween(2, 16);
+        indexDocs("metrics-foo", "@timestamp1", numDocsRolledFoo);
+
+        SearchRequest searchRequest = new SearchRequest("*");
+        SearchResponse searchResponse = client().search(searchRequest).actionGet();
+        assertThat(searchResponse.getHits().getTotalHits().value, is((long) numDocsBar + numDocsFoo + numDocsRolledFoo));
+    }
+
     private static void assertBackingIndex(String backingIndex, String timestampFieldPathInMapping) {
         assertBackingIndex(backingIndex, timestampFieldPathInMapping, Map.of("type", "date"));
     }

server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java

@@ -172,7 +172,6 @@ public class IndexNameExpressionResolver {
         }
         Metadata metadata = context.getState().metadata();
         IndicesOptions options = context.getOptions();
-        final boolean failClosed = options.forbidClosedIndices() && options.ignoreUnavailable() == false;
         // If only one index is specified then whether we fail a request if an index is missing depends on the allow_no_indices
         // option. At some point we should change this, because there shouldn't be a reason why whether a single index
         // or multiple indices are specified yield different behaviour.
@@ -256,20 +255,8 @@ public class IndexNameExpressionResolver {
                 }
 
                 for (IndexMetadata index : indexAbstraction.getIndices()) {
-                    if (index.getState() == IndexMetadata.State.CLOSE) {
-                        if (failClosed) {
-                            throw new IndexClosedException(index.getIndex());
-                        } else {
-                            if (options.forbidClosedIndices() == false && addIndex(index, context)) {
-                                concreteIndices.add(index.getIndex());
-                            }
-                        }
-                    } else if (index.getState() == IndexMetadata.State.OPEN) {
-                        if (addIndex(index, context)) {
-                            concreteIndices.add(index.getIndex());
-                        }
-                    } else {
-                        throw new IllegalStateException("index state [" + index.getState() + "] not supported");
+                    if (shouldTrackConcreteIndex(context, options, index)) {
+                        concreteIndices.add(index.getIndex());
                     }
                 }
             }
@@ -283,6 +270,20 @@ public class IndexNameExpressionResolver {
         return concreteIndices.toArray(new Index[concreteIndices.size()]);
     }
 
+    private static boolean shouldTrackConcreteIndex(Context context, IndicesOptions options, IndexMetadata index) {
+        if (index.getState() == IndexMetadata.State.CLOSE) {
+            if (options.forbidClosedIndices() && options.ignoreUnavailable() == false) {
+                throw new IndexClosedException(index.getIndex());
+            } else {
+                return options.forbidClosedIndices() == false && addIndex(index, context);
+            }
+        } else if (index.getState() == IndexMetadata.State.OPEN) {
+            return addIndex(index, context);
+        } else {
+            throw new IllegalStateException("index state [" + index.getState() + "] not supported");
+        }
+    }
+
     private static boolean addIndex(IndexMetadata metadata, Context context) {
         // This used to check the `index.search.throttled` setting, but we eventually decided that it was
         // trappy to hide throttled indices by default. In order to avoid breaking backward compatibility,
@@ -731,7 +732,21 @@ public class IndexNameExpressionResolver {
                 if (context.includeDataStreams() == false && metadata.dataStreams().isEmpty() == false) {
                     throw dataStreamsNotSupportedException(expressions.toString());
                 }
-                return resolveEmptyOrTrivialWildcard(options, metadata);
+
+                List<String> resolvedExpressions = resolveEmptyOrTrivialWildcard(options, metadata);
+                if (context.includeDataStreams()) {
+                    final IndexMetadata.State excludeState = excludeState(options);
+                    final Map<String, IndexAbstraction> dataStreamsAbstractions = metadata.getIndicesLookup().entrySet()
+                        .stream()
+                        .filter(entry -> entry.getValue().getType() == IndexAbstraction.Type.DATA_STREAM)
+                        .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+                    // dedup backing indices if expand hidden indices option is true
+                    Set<String> resolvedIncludingDataStreams = new HashSet<>(resolvedExpressions);
+                    resolvedIncludingDataStreams.addAll(expand(context, excludeState, dataStreamsAbstractions,
+                        expressions.isEmpty() ? "_all" : expressions.get(0), options.expandWildcardsHidden()));
+                    return new ArrayList<>(resolvedIncludingDataStreams);
+                }
+                return resolvedExpressions;
             }
 
             Set<String> result = innerResolve(context, expressions, options, metadata);

server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java

@@ -1916,6 +1916,17 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
             assertThat(result[2].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStream2, 1)));
             assertThat(result[3].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStream2, 2)));;
         }
+        {
+            IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN;
+            Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true,
+                randomFrom(new String[]{"*"}, new String[]{"_all"}, new String[0]));
+            Arrays.sort(result, Comparator.comparing(Index::getName));
+            assertThat(result.length, equalTo(4));
+            assertThat(result[0].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStream1, 1)));
+            assertThat(result[1].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStream1, 2)));
+            assertThat(result[2].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStream2, 1)));
+            assertThat(result[3].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStream2, 2)));;
+        }
         {
             IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN;
             Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "logs-m*");
@@ -1933,6 +1944,39 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
         }
     }
 
+    public void testDataStreamsWithClosedBackingIndicesAndWildcardExpressions() {
+        final String dataStream1 = "logs-mysql";
+        final String dataStream2 = "logs-redis";
+        IndexMetadata index1 = createBackingIndex(dataStream1, 1).state(State.CLOSE).build();
+        IndexMetadata index2 = createBackingIndex(dataStream1, 2).build();
+        IndexMetadata index3 = createBackingIndex(dataStream2, 1).state(State.CLOSE).build();
+        IndexMetadata index4 = createBackingIndex(dataStream2, 2).build();
+        Metadata.Builder mdBuilder = Metadata.builder()
+            .put(index1, false)
+            .put(index2, false)
+            .put(index3, false)
+            .put(index4, false)
+            .put(new DataStream(dataStream1, createTimestampField("@timestamp"), List.of(index1.getIndex(), index2.getIndex())))
+            .put(new DataStream(dataStream2, createTimestampField("@timestamp"), List.of(index3.getIndex(), index4.getIndex())));
+
+        ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
+        IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN;
+        {
+            Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "logs-*");
+            Arrays.sort(result, Comparator.comparing(Index::getName));
+            assertThat(result.length, equalTo(2));
+            assertThat(result[0].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStream1, 2)));
+            assertThat(result[1].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStream2, 2)));;
+        }
+        {
+            Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "*");
+            Arrays.sort(result, Comparator.comparing(Index::getName));
+            assertThat(result.length, equalTo(2));
+            assertThat(result[0].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStream1, 2)));
+            assertThat(result[1].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStream2, 2)));;
+        }
+    }
+
     public void testDataStreamsWithRegularIndexAndAlias() {
         final String dataStream1 = "logs-foobar";
         IndexMetadata index1 = createBackingIndex(dataStream1, 1).build();

server/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java

@@ -32,9 +32,12 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Set;
 
+import static org.elasticsearch.cluster.DataStreamTestHelper.createBackingIndex;
+import static org.elasticsearch.cluster.DataStreamTestHelper.createTimestampField;
 import static org.elasticsearch.common.util.set.Sets.newHashSet;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
 
 public class WildcardExpressionResolverTests extends ESTestCase {
     public void testConvertWildcardsJustIndicesTests() {
@@ -207,6 +210,77 @@ public class WildcardExpressionResolverTests extends ESTestCase {
         }
     }
 
+    public void testResolveDataStreams() {
+        String dataStreamName = "foo_logs";
+        IndexMetadata firstBackingIndexMetadata = createBackingIndex(dataStreamName, 1).build();
+        IndexMetadata secondBackingIndexMetadata = createBackingIndex(dataStreamName, 2).build();
+
+        Metadata.Builder mdBuilder = Metadata.builder()
+            .put(indexBuilder("foo_foo").state(State.OPEN))
+            .put(indexBuilder("bar_bar").state(State.OPEN))
+            .put(indexBuilder("foo_index").state(State.OPEN).putAlias(AliasMetadata.builder("foo_alias")))
+            .put(indexBuilder("bar_index").state(State.OPEN).putAlias(AliasMetadata.builder("foo_alias")))
+            .put(firstBackingIndexMetadata, true)
+            .put(secondBackingIndexMetadata, true)
+            .put(new DataStream(dataStreamName, createTimestampField("timestamp"),
+                List.of(firstBackingIndexMetadata.getIndex(), secondBackingIndexMetadata.getIndex())));
+
+        ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
+
+        IndexNameExpressionResolver.WildcardExpressionResolver resolver = new IndexNameExpressionResolver.WildcardExpressionResolver();
+
+        {
+            IndicesOptions indicesAndAliasesOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), true, false, true, false,
+                false, false);
+            IndexNameExpressionResolver.Context indicesAndAliasesContext =
+                new IndexNameExpressionResolver.Context(state, indicesAndAliasesOptions);
+
+            // data streams are not included but expression matches the data stream
+            IllegalArgumentException illegalArgumentException = expectThrows(IllegalArgumentException.class,
+                () -> resolver.resolve(indicesAndAliasesContext, Collections.singletonList("foo_*")));
+            assertThat(illegalArgumentException.getMessage(), is("The provided expression [foo_*] matches a data stream, specify the " +
+                "corresponding concrete indices instead."));
+
+            // data streams are not included and expression doesn't match the data steram
+            List<String> indices = resolver.resolve(indicesAndAliasesContext, Collections.singletonList("bar_*"));
+            assertThat(indices, containsInAnyOrder("bar_bar", "bar_index"));
+        }
+
+        {
+            IndicesOptions indicesAndAliasesOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), true, false, true, false,
+                false, false);
+            IndexNameExpressionResolver.Context indicesAliasesAndDataStreamsContext = new IndexNameExpressionResolver.Context(state,
+                indicesAndAliasesOptions, false, false, true);
+
+            // data stream's corresponding backing indices are resolved
+            List<String> indices = resolver.resolve(indicesAliasesAndDataStreamsContext, Collections.singletonList("foo_*"));
+            assertThat(indices, containsInAnyOrder("foo_index", "bar_index", "foo_foo", ".ds-foo_logs-000001",
+                ".ds-foo_logs-000002"));
+
+            // include all wildcard adds the data stream's backing indices
+            indices = resolver.resolve(indicesAliasesAndDataStreamsContext, Collections.singletonList("*"));
+            assertThat(indices, containsInAnyOrder("foo_index", "bar_index", "foo_foo", "bar_bar", ".ds-foo_logs-000001",
+                ".ds-foo_logs-000002"));
+        }
+
+        {
+            IndicesOptions indicesAliasesAndExpandHiddenOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), true, false,
+                true, true, false, false, false);
+            IndexNameExpressionResolver.Context indicesAliasesDataStreamsAndHiddenIndices = new IndexNameExpressionResolver.Context(state,
+                indicesAliasesAndExpandHiddenOptions, false, false, true);
+
+            // data stream's corresponding backing indices are resolved
+            List<String> indices = resolver.resolve(indicesAliasesDataStreamsAndHiddenIndices, Collections.singletonList("foo_*"));
+            assertThat(indices, containsInAnyOrder("foo_index", "bar_index", "foo_foo", ".ds-foo_logs-000001",
+                ".ds-foo_logs-000002"));
+
+            // include all wildcard adds the data stream's backing indices
+            indices = resolver.resolve(indicesAliasesDataStreamsAndHiddenIndices, Collections.singletonList("*"));
+            assertThat(indices, containsInAnyOrder("foo_index", "bar_index", "foo_foo", "bar_bar", ".ds-foo_logs-000001",
+                ".ds-foo_logs-000002"));
+        }
+    }
+
     public void testMatchesConcreteIndicesWildcardAndAliases() {
         Metadata.Builder mdBuilder = Metadata.builder()
                 .put(indexBuilder("foo_foo").state(State.OPEN))