GET /_all should return hidden indices with visible aliases (#106975)

GET /_all should return hidden indices if they are accessible through a
visible alias. This is currently the behavior when resolution occurs in the 
security layer. This changes adds this behavior to name resolution
when security is not used.

docs/changelog/106975.yaml

@@ -0,0 +1,5 @@
+pr: 106975
+summary: GET /_all should return hidden indices with visible aliases
+area: Indices APIs
+type: bug
+issues: []

server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java

@@ -1246,32 +1246,36 @@ public class IndexNameExpressionResolver {
         }
 
         /**
-         * Returns all the indices and all the datastreams, considering the open/closed, system, and hidden context parameters.
+         * Returns all the indices, datastreams, and aliases, considering the open/closed, system, and hidden context parameters.
          * Depending on the context, returns the names of the datastreams themselves or their backing indices.
          */
         public static Collection<String> resolveAll(Context context) {
-            List<String> resolvedExpressions = resolveEmptyOrTrivialWildcard(context);
-            if (context.includeDataStreams() == false) {
-                return resolvedExpressions;
-            } else {
-                Stream<IndexAbstraction> dataStreamsAbstractions = context.getState()
-                    .metadata()
-                    .getIndicesLookup()
-                    .values()
-                    .stream()
-                    .filter(indexAbstraction -> indexAbstraction.getType() == Type.DATA_STREAM)
-                    .filter(
-                        indexAbstraction -> indexAbstraction.isSystem() == false
-                            || context.systemIndexAccessPredicate.test(indexAbstraction.getName())
-                    );
-                if (context.getOptions().expandWildcardsHidden() == false) {
-                    dataStreamsAbstractions = dataStreamsAbstractions.filter(indexAbstraction -> indexAbstraction.isHidden() == false);
-                }
-                // dedup backing indices if expand hidden indices option is true
-                Set<String> resolvedIncludingDataStreams = expandToOpenClosed(context, dataStreamsAbstractions).collect(Collectors.toSet());
-                resolvedIncludingDataStreams.addAll(resolvedExpressions);
-                return resolvedIncludingDataStreams;
+            List<String> concreteIndices = resolveEmptyOrTrivialWildcard(context);
+
+            if (context.includeDataStreams() == false && context.getOptions().ignoreAliases()) {
+                return concreteIndices;
             }
+
+            Stream<IndexAbstraction> ias = context.getState()
+                .metadata()
+                .getIndicesLookup()
+                .values()
+                .stream()
+                .filter(ia -> context.getOptions().expandWildcardsHidden() || ia.isHidden() == false)
+                .filter(ia -> shouldIncludeIfDataStream(ia, context) || shouldIncludeIfAlias(ia, context))
+                .filter(ia -> ia.isSystem() == false || context.systemIndexAccessPredicate.test(ia.getName()));
+
+            Set<String> resolved = expandToOpenClosed(context, ias).collect(Collectors.toSet());
+            resolved.addAll(concreteIndices);
+            return resolved;
+        }
+
+        private static boolean shouldIncludeIfDataStream(IndexAbstraction ia, IndexNameExpressionResolver.Context context) {
+            return context.includeDataStreams() && ia.getType() == Type.DATA_STREAM;
+        }
+
+        private static boolean shouldIncludeIfAlias(IndexAbstraction ia, IndexNameExpressionResolver.Context context) {
+            return context.getOptions().ignoreAliases() == false && ia.getType() == Type.ALIAS;
         }
 
         /**

server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java

@@ -1217,9 +1217,9 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
             indexNames = indexNameExpressionResolver.concreteIndexNames(state, includeHiddenOptions, visibleAlias);
             assertThat(Arrays.asList(indexNames), containsInAnyOrder(visibleIndex, hiddenIndex));
 
-            // A total wildcards does not resolve the hidden index in this case
+            // total wildcards should also resolve both visible and hidden indices if there is a visible alias
             indexNames = indexNameExpressionResolver.concreteIndexNames(state, excludeHiddenOptions, "*");
-            assertThat(Arrays.asList(indexNames), containsInAnyOrder(visibleIndex));
+            assertThat(Arrays.asList(indexNames), containsInAnyOrder(visibleIndex, hiddenIndex));
         }
 
         {

server/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java

@@ -280,6 +280,129 @@ public class WildcardExpressionResolverTests extends ESTestCase {
         assertThat(IndexNameExpressionResolver.resolveExpressions(noExpandContext, "_all").size(), equalTo(0));
     }
 
+    public void testAllAliases() {
+        {
+            // hidden index with hidden alias should not be returned
+            Metadata.Builder mdBuilder = Metadata.builder()
+                .put(
+                    indexBuilder("index-hidden-alias", true) // index hidden
+                        .state(State.OPEN)
+                        .putAlias(AliasMetadata.builder("alias-hidden").isHidden(true)) // alias hidden
+                );
+
+            ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
+
+            IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(
+                state,
+                IndicesOptions.lenientExpandOpen(), // don't include hidden
+                SystemIndexAccessLevel.NONE
+            );
+            assertThat(newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context)), equalTo(newHashSet()));
+        }
+
+        {
+            // hidden index with visible alias should be returned
+            Metadata.Builder mdBuilder = Metadata.builder()
+                .put(
+                    indexBuilder("index-visible-alias", true) // index hidden
+                        .state(State.OPEN)
+                        .putAlias(AliasMetadata.builder("alias-visible").isHidden(false)) // alias visible
+                );
+
+            ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
+
+            IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(
+                state,
+                IndicesOptions.lenientExpandOpen(), // don't include hidden
+                SystemIndexAccessLevel.NONE
+            );
+            assertThat(
+                newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context)),
+                equalTo(newHashSet("index-visible-alias"))
+            );
+        }
+    }
+
+    public void testAllDataStreams() {
+
+        String dataStreamName = "foo_logs";
+        long epochMillis = randomLongBetween(1580536800000L, 1583042400000L);
+        IndexMetadata firstBackingIndexMetadata = createBackingIndex(dataStreamName, 1, epochMillis).build();
+
+        IndicesOptions indicesAndAliasesOptions = IndicesOptions.fromOptions(
+            randomBoolean(),
+            randomBoolean(),
+            true,
+            false,
+            true,
+            false,
+            false,
+            false
+        );
+
+        {
+            // visible data streams should be returned by _all even show backing indices are hidden
+            Metadata.Builder mdBuilder = Metadata.builder()
+                .put(firstBackingIndexMetadata, true)
+                .put(DataStreamTestHelper.newInstance(dataStreamName, List.of(firstBackingIndexMetadata.getIndex())));
+
+            ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
+
+            IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(
+                state,
+                indicesAndAliasesOptions,
+                false,
+                false,
+                true,
+                SystemIndexAccessLevel.NONE,
+                NONE,
+                NONE
+            );
+
+            assertThat(
+                newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context)),
+                equalTo(newHashSet(DataStream.getDefaultBackingIndexName("foo_logs", 1, epochMillis)))
+            );
+        }
+
+        {
+            // if data stream itself is hidden, backing indices should not be returned
+            boolean hidden = true;
+            var dataStream = new DataStream(
+                dataStreamName,
+                List.of(firstBackingIndexMetadata.getIndex()),
+                1,
+                null,
+                hidden,
+                false,
+                false,
+                false,
+                null,
+                null,
+                false,
+                List.of(),
+                null
+            );
+
+            Metadata.Builder mdBuilder = Metadata.builder().put(firstBackingIndexMetadata, true).put(dataStream);
+
+            ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
+
+            IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(
+                state,
+                indicesAndAliasesOptions,
+                false,
+                false,
+                true,
+                SystemIndexAccessLevel.NONE,
+                NONE,
+                NONE
+            );
+
+            assertThat(newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context)), equalTo(newHashSet()));
+        }
+    }
+
     public void testResolveEmpty() {
         Metadata.Builder mdBuilder = Metadata.builder()
             .put(