Add indices and filter information to search shards api output (#21738)

Add indices and filter information to search shards api output

The search shards api returns info about which shards are going to be hit by executing a search with provided parameters: indices, routing, preference. Indices can also be aliases, which can also hold filters. The output includes an array of shards and a summary of all the nodes the shards are allocated on. This commit adds a new indices section to the search shards output that includes one entry per index, where each index can be associated with an optional filter in case the index was hit through a filtered alias.

This is relevant since we have moved parsing of alias filters to the coordinating node.

Relates to #20916

buildSrc/src/main/resources/checkstyle_suppressions.xml

@@ -40,7 +40,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]settings[/\\]SettingsUpdater.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]shards[/\\]ClusterSearchShardsAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]shards[/\\]ClusterSearchShardsRequestBuilder.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]shards[/\\]TransportClusterSearchShardsAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]snapshots[/\\]create[/\\]CreateSnapshotRequestBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]snapshots[/\\]create[/\\]TransportCreateSnapshotAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]snapshots[/\\]delete[/\\]DeleteSnapshotRequestBuilder.java" checks="LineLength" />

core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsResponse.java

@@ -19,24 +19,37 @@
 
 package org.elasticsearch.action.admin.cluster.shards;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.search.internal.AliasFilter;
 
 import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
 
 public class ClusterSearchShardsResponse extends ActionResponse implements ToXContent {
+    public static final Version V_5_1_0_UNRELEASED = Version.fromId(5010099);
 
     private ClusterSearchShardsGroup[] groups;
     private DiscoveryNode[] nodes;
+    private Map<String, AliasFilter> indicesAndFilters;
 
     ClusterSearchShardsResponse() {
 
     }
 
+    ClusterSearchShardsResponse(ClusterSearchShardsGroup[] groups, DiscoveryNode[] nodes,
+                                       Map<String, AliasFilter> indicesAndFilters) {
+        this.groups = groups;
+        this.nodes = nodes;
+        this.indicesAndFilters = indicesAndFilters;
+    }
+
     public ClusterSearchShardsGroup[] getGroups() {
         return groups;
     }
@@ -45,9 +58,8 @@ public class ClusterSearchShardsResponse extends ActionResponse implements ToXCo
         return nodes;
     }
 
-    public ClusterSearchShardsResponse(ClusterSearchShardsGroup[] groups, DiscoveryNode[] nodes) {
-        this.groups = groups;
-        this.nodes = nodes;
+    public Map<String, AliasFilter> getIndicesAndFilters() {
+        return indicesAndFilters;
     }
 
     @Override
@@ -61,7 +73,15 @@ public class ClusterSearchShardsResponse extends ActionResponse implements ToXCo
         for (int i = 0; i < nodes.length; i++) {
             nodes[i] = new DiscoveryNode(in);
         }
-
+        if (in.getVersion().onOrAfter(V_5_1_0_UNRELEASED)) {
+            int size = in.readVInt();
+            indicesAndFilters = new HashMap<>();
+            for (int i = 0; i < size; i++) {
+                String index = in.readString();
+                AliasFilter aliasFilter = new AliasFilter(in);
+                indicesAndFilters.put(index, aliasFilter);
+            }
+        }
     }
 
     @Override
@@ -75,6 +95,13 @@ public class ClusterSearchShardsResponse extends ActionResponse implements ToXCo
         for (DiscoveryNode node : nodes) {
             node.writeTo(out);
         }
+        if (out.getVersion().onOrAfter(V_5_1_0_UNRELEASED)) {
+            out.writeVInt(indicesAndFilters.size());
+            for (Map.Entry<String, AliasFilter> entry : indicesAndFilters.entrySet()) {
+                out.writeString(entry.getKey());
+                entry.getValue().writeTo(out);
+            }
+        }
     }
 
     @Override
@@ -84,6 +111,20 @@ public class ClusterSearchShardsResponse extends ActionResponse implements ToXCo
             node.toXContent(builder, params);
         }
         builder.endObject();
+        if (indicesAndFilters != null) {
+            builder.startObject("indices");
+            for (Map.Entry<String, AliasFilter> entry : indicesAndFilters.entrySet()) {
+                String index = entry.getKey();
+                builder.startObject(index);
+                AliasFilter aliasFilter = entry.getValue();
+                if (aliasFilter.getQueryBuilder() != null) {
+                    builder.field("filter");
+                    aliasFilter.getQueryBuilder().toXContent(builder, params);
+                }
+                builder.endObject();
+            }
+            builder.endObject();
+        }
         builder.startArray("shards");
         for (ClusterSearchShardsGroup group : groups) {
             group.toXContent(builder, params);

core/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java

@@ -33,21 +33,29 @@ import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 
-public class TransportClusterSearchShardsAction extends TransportMasterNodeReadAction<ClusterSearchShardsRequest, ClusterSearchShardsResponse> {
+public class TransportClusterSearchShardsAction extends
+        TransportMasterNodeReadAction<ClusterSearchShardsRequest, ClusterSearchShardsResponse> {
+
+    private final IndicesService indicesService;
 
     @Inject
     public TransportClusterSearchShardsAction(Settings settings, TransportService transportService, ClusterService clusterService,
-                                              ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
-        super(settings, ClusterSearchShardsAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, ClusterSearchShardsRequest::new);
+                                              IndicesService indicesService, ThreadPool threadPool, ActionFilters actionFilters,
+                                              IndexNameExpressionResolver indexNameExpressionResolver) {
+        super(settings, ClusterSearchShardsAction.NAME, transportService, clusterService, threadPool, actionFilters,
+                indexNameExpressionResolver, ClusterSearchShardsRequest::new);
+        this.indicesService = indicesService;
     }
 
     @Override
@@ -58,7 +66,8 @@ public class TransportClusterSearchShardsAction extends TransportMasterNodeReadA
 
     @Override
     protected ClusterBlockException checkBlock(ClusterSearchShardsRequest request, ClusterState state) {
-        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ, indexNameExpressionResolver.concreteIndexNames(state, request));
+        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ,
+                indexNameExpressionResolver.concreteIndexNames(state, request));
     }
 
     @Override
@@ -67,12 +76,20 @@ public class TransportClusterSearchShardsAction extends TransportMasterNodeReadA
     }
 
     @Override
-    protected void masterOperation(final ClusterSearchShardsRequest request, final ClusterState state, final ActionListener<ClusterSearchShardsResponse> listener) {
+    protected void masterOperation(final ClusterSearchShardsRequest request, final ClusterState state,
+                                   final ActionListener<ClusterSearchShardsResponse> listener) {
         ClusterState clusterState = clusterService.state();
         String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, request);
         Map<String, Set<String>> routingMap = indexNameExpressionResolver.resolveSearchRouting(state, request.routing(), request.indices());
+        Map<String, AliasFilter> indicesAndFilters = new HashMap<>();
+        for (String index : concreteIndices) {
+            AliasFilter aliasFilter = indicesService.buildAliasFilter(clusterState, index, request.indices());
+            indicesAndFilters.put(index, aliasFilter);
+        }
+
         Set<String> nodeIds = new HashSet<>();
-        GroupShardsIterator groupShardsIterator = clusterService.operationRouting().searchShards(clusterState, concreteIndices, routingMap, request.preference());
+        GroupShardsIterator groupShardsIterator = clusterService.operationRouting().searchShards(clusterState, concreteIndices,
+                routingMap, request.preference());
         ShardRouting shard;
         ClusterSearchShardsGroup[] groupResponses = new ClusterSearchShardsGroup[groupShardsIterator.size()];
         int currentGroup = 0;
@@ -92,6 +109,6 @@ public class TransportClusterSearchShardsAction extends TransportMasterNodeReadA
         for (String nodeId : nodeIds) {
             nodes[currentNode++] = clusterState.getNodes().get(nodeId);
         }
-        listener.onResponse(new ClusterSearchShardsResponse(groupResponses, nodes));
+        listener.onResponse(new ClusterSearchShardsResponse(groupResponses, nodes, indicesAndFilters));
     }
 }

core/src/test/java/org/elasticsearch/VersionTests.java

@@ -20,6 +20,7 @@
 package org.elasticsearch;
 
 import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsRequest;
+import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
@@ -291,6 +292,7 @@ public class VersionTests extends ESTestCase {
         assertUnknownVersion(Script.V_5_1_0_UNRELEASED);
         // once we released 5.0.0 and it's added to Version.java we need to remove this constant
         assertUnknownVersion(ClusterSearchShardsRequest.V_5_1_0_UNRELEASED);
+        assertUnknownVersion(ClusterSearchShardsResponse.V_5_1_0_UNRELEASED);
     }
 
     public static void assertUnknownVersion(Version version) {

core/src/test/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequestTests.java

@@ -54,7 +54,7 @@ public class ClusterSearchShardsRequestTests extends ESTestCase {
             request.routing(routings);
         }
 
-        Version version = VersionUtils.randomVersion(random());
+        Version version = VersionUtils.randomVersionBetween(random(), Version.V_5_0_0, Version.CURRENT);
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.setVersion(version);
             request.writeTo(out);

core/src/test/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsResponseTests.java

@@ -0,0 +1,105 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.cluster.shards;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.TestShardRouting;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.index.query.RandomQueryBuilder;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.search.internal.AliasFilter;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.VersionUtils;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class ClusterSearchShardsResponseTests extends ESTestCase {
+
+    public void testSerialization() throws Exception {
+        Map<String, AliasFilter> indicesAndFilters = new HashMap<>();
+        Set<DiscoveryNode> nodes = new HashSet<>();
+        int numShards = randomIntBetween(1, 10);
+        ClusterSearchShardsGroup[] clusterSearchShardsGroups = new ClusterSearchShardsGroup[numShards];
+        for (int i = 0; i < numShards; i++) {
+            String index = randomAsciiOfLengthBetween(3, 10);
+            ShardId shardId = new ShardId(index, randomAsciiOfLength(12), i);
+            String nodeId = randomAsciiOfLength(10);
+            ShardRouting shardRouting = TestShardRouting.newShardRouting(shardId, nodeId, randomBoolean(), ShardRoutingState.STARTED);
+            clusterSearchShardsGroups[i] = new ClusterSearchShardsGroup(shardId, new ShardRouting[]{shardRouting});
+            DiscoveryNode node = new DiscoveryNode(shardRouting.currentNodeId(),
+                    new TransportAddress(TransportAddress.META_ADDRESS, randomInt(0xFFFF)), VersionUtils.randomVersion(random()));
+            nodes.add(node);
+            AliasFilter aliasFilter;
+            if (randomBoolean()) {
+                aliasFilter = new AliasFilter(RandomQueryBuilder.createQuery(random()), "alias-" + index);
+            } else {
+                aliasFilter = new AliasFilter(null, Strings.EMPTY_ARRAY);
+            }
+            indicesAndFilters.put(index, aliasFilter);
+        }
+        ClusterSearchShardsResponse clusterSearchShardsResponse = new ClusterSearchShardsResponse(clusterSearchShardsGroups,
+                nodes.toArray(new DiscoveryNode[nodes.size()]), indicesAndFilters);
+
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
+        entries.addAll(searchModule.getNamedWriteables());
+        NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(entries);
+        Version version = VersionUtils.randomVersionBetween(random(), Version.V_5_0_0, Version.CURRENT);
+        try(BytesStreamOutput out = new BytesStreamOutput()) {
+            out.setVersion(version);
+            clusterSearchShardsResponse.writeTo(out);
+            try(StreamInput in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), namedWriteableRegistry)) {
+                in.setVersion(version);
+                ClusterSearchShardsResponse deserialized = new ClusterSearchShardsResponse();
+                deserialized.readFrom(in);
+                assertArrayEquals(clusterSearchShardsResponse.getNodes(), deserialized.getNodes());
+                assertEquals(clusterSearchShardsResponse.getGroups().length, deserialized.getGroups().length);
+                for (int i = 0; i < clusterSearchShardsResponse.getGroups().length; i++) {
+                    ClusterSearchShardsGroup clusterSearchShardsGroup = clusterSearchShardsResponse.getGroups()[i];
+                    ClusterSearchShardsGroup deserializedGroup = deserialized.getGroups()[i];
+                    assertEquals(clusterSearchShardsGroup.getShardId(), deserializedGroup.getShardId());
+                    assertEquals(clusterSearchShardsGroup.getIndex(), deserializedGroup.getIndex());
+                    assertArrayEquals(clusterSearchShardsGroup.getShards(), deserializedGroup.getShards());
+                }
+                if (version.onOrAfter(ClusterSearchShardsResponse.V_5_1_0_UNRELEASED)) {
+                    assertEquals(clusterSearchShardsResponse.getIndicesAndFilters(), deserialized.getIndicesAndFilters());
+                } else {
+                    assertNull(deserialized.getIndicesAndFilters());
+                }
+            }
+        }
+    }
+}

docs/reference/search/search-shards.asciidoc

@@ -3,7 +3,8 @@
 
 The search shards api returns the indices and shards that a search request would
 be executed against. This can give useful feedback for working out issues or
-planning optimizations with routing and shard preferences.
+planning optimizations with routing and shard preferences. When filtered aliases
+are used, the filter is returned as part of the `indices` section.
 
 The `index` may be a single value, or comma-separated.
 
@@ -25,6 +26,9 @@ This will yield the following result:
 --------------------------------------------------
 {
   "nodes": ...,
+  "indices" : {
+    "twitter": { }
+  },
   "shards": [
     [
       {
@@ -107,6 +111,9 @@ This will yield the following result:
 --------------------------------------------------
 {
   "nodes": ...,
+  "indices" : {
+      "twitter": { }
+  },
   "shards": [
     [
       {

rest-api-spec/src/main/resources/rest-api-spec/test/search_shards/10_basic.yaml

@@ -10,3 +10,47 @@
         routing: foo
 
   - match: { shards.0.0.index: test_1 }
+
+---
+"Search shards aliases with and without filters":
+  - skip:
+      version: " - 5.0.99"
+      reason: indices section was added in 5.1.0
+
+  - do:
+      indices.create:
+        index: test_index
+        body:
+          settings:
+            index:
+              number_of_shards: 1
+              number_of_replicas: 0
+          mappings:
+            type_1:
+              properties:
+                field:
+                  type: text
+          aliases:
+            test_alias_1: {}
+            test_alias_2:
+              filter:
+                term:
+                  field : value
+
+  - do:
+      search_shards:
+        index:  test_alias_1
+
+  - length: { shards: 1 }
+  - match: { shards.0.0.index: test_index }
+  - is_true: indices.test_index
+  - is_false: indices.test_index.filter
+
+  - do:
+      search_shards:
+        index:  test_alias_2
+
+  - length: { shards: 1 }
+  - match: { shards.0.0.index: test_index }
+  - match: { indices.test_index: {filter: { term : { field: { value: value, boost: 1.0}}}}}
+