Get hidden indices stats in `GET _cat/shards` (#86601)

Passes a permissive `IndicesOptions` to the indices stats request used
within `GET _cat/shards` so that it retrieves stats for hidden indices
by default.

Also passes the same `IndicesOptions` to the cluster state request so
that the two requests get consistent sets of indices.

Also parallelises the two requests since there's no dependency between
them.

Closes #84656

Also relates #32238 since the more permissive `IndicesOptions` used here
permits closed indices, which means `GET _cat/shards` will not throw an
exception in security-enabled clusters containing closed indices behind
aliases.

docs/changelog/86601.yaml

@@ -0,0 +1,6 @@
+pr: 86601
+summary: Get hidden indices stats in `GET _cat/shards`
+area: CAT APIs
+type: bug
+issues:
+ - 84656

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.shards/10_basic.yml

@@ -216,3 +216,28 @@
             /^ foo \s+ 0\n
                bar \s+ 1\n
             $/
+
+
+---
+"Test cat shards with hidden indices":
+  - do:
+      indices.create:
+        index: foo
+        body:
+          settings:
+            number_of_shards: "1"
+            number_of_replicas: "0"
+            hidden: "true"
+
+  - do:
+      index:
+        index: foo
+        body: { test: foo }
+        refresh: true
+
+  - do:
+      cat.shards:
+        h: [index, docs]
+
+  - match:
+      $body: /foo \s+ 1\n/

server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java

@@ -14,12 +14,14 @@ import org.elasticsearch.action.admin.indices.stats.CommonStats;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.admin.indices.stats.ShardStats;
+import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.util.concurrent.ListenableFuture;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.bulk.stats.BulkStats;
 import org.elasticsearch.index.cache.query.QueryCacheStats;
@@ -38,7 +40,6 @@ import org.elasticsearch.index.store.StoreStats;
 import org.elasticsearch.index.warmer.WarmerStats;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
-import org.elasticsearch.rest.action.RestActionListener;
 import org.elasticsearch.rest.action.RestResponseListener;
 import org.elasticsearch.search.suggest.completion.CompletionStats;
 
@@ -75,23 +76,30 @@ public class RestShardsAction extends AbstractCatAction {
     @Override
     public RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) {
         final String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
-        final ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
+
+        final var clusterStateRequest = new ClusterStateRequest();
         clusterStateRequest.masterNodeTimeout(request.paramAsTime("master_timeout", clusterStateRequest.masterNodeTimeout()));
-        clusterStateRequest.clear().nodes(true).routingTable(true).indices(indices);
-        return channel -> client.admin().cluster().state(clusterStateRequest, new RestActionListener<ClusterStateResponse>(channel) {
-            @Override
-            public void processResponse(final ClusterStateResponse clusterStateResponse) {
-                IndicesStatsRequest indicesStatsRequest = new IndicesStatsRequest();
-                indicesStatsRequest.all();
-                indicesStatsRequest.indices(indices);
-                client.admin().indices().stats(indicesStatsRequest, new RestResponseListener<IndicesStatsResponse>(channel) {
-                    @Override
-                    public RestResponse buildResponse(IndicesStatsResponse indicesStatsResponse) throws Exception {
-                        return RestTable.buildResponse(buildTable(request, clusterStateResponse, indicesStatsResponse), channel);
-                    }
-                });
-            }
-        });
+        clusterStateRequest.clear().nodes(true).routingTable(true).indices(indices).indicesOptions(IndicesOptions.strictExpandHidden());
+
+        return channel -> {
+            final var clusterStateFuture = new ListenableFuture<ClusterStateResponse>();
+            client.admin().cluster().state(clusterStateRequest, clusterStateFuture);
+            client.admin()
+                .indices()
+                .stats(
+                    new IndicesStatsRequest().all().indices(indices).indicesOptions(IndicesOptions.strictExpandHidden()),
+                    new RestResponseListener<Table>(channel) {
+                        @Override
+                        public RestResponse buildResponse(Table table) throws Exception {
+                            return RestTable.buildResponse(table, channel);
+                        }
+                    }.delegateFailure(
+                        (delegate, indicesStatsResponse) -> clusterStateFuture.addListener(
+                            delegate.map(clusterStateResponse -> buildTable(request, clusterStateResponse, indicesStatsResponse))
+                        )
+                    )
+                );
+        };
     }
 
     @Override