Expose `max_concurrent_shard_requests` in `_msearch` (#33016)

Today `_msearch` doesn't allow modifying the `max_concurrent_shard_requests`
per sub search request. This change adds support for setting this parameter on
all sub-search requests in an `_msearch`.

Relates to #31877

docs/reference/search/multi-search.asciidoc

@@ -86,6 +86,16 @@ The msearch's `max_concurrent_searches` request parameter can be used to control
 the maximum number of concurrent searches the multi search api will execute.
 This default is based on the number of data nodes and the default search thread pool size.
 
+The request parameter `max_concurrent_shard_requests` can be used to control the
+maximum number of concurrent shard requests the each sub search request will execute.
+This parameter should be used to protect a single request from overloading a cluster
+(e.g., a default request will hit all indices in a cluster which could cause shard request rejections
+if the number of shards per node is high). This default is based on the number of
+data nodes in the cluster but at most `256`.In certain scenarios parallelism isn't achieved through
+concurrent request such that this protection will result in poor performance. For
+instance in an environment where only a very low number of concurrent search requests are expected
+it might help to increase this value to a higher number.
+
 [float]
 [[msearch-security]]
 === Security

rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json

@@ -33,6 +33,11 @@
           "type" : "number",
           "description" : "A threshold that enforces a pre-filter roundtrip to prefilter search shards based on query rewriting if the number of shards the search request expands to exceeds the threshold. This filter roundtrip can limit the number of shards significantly if for instance a shard can not match any documents based on it's rewrite method ie. if date filters are mandatory to match but the shard bounds and the query are disjoint.",
           "default" : 128
+        },
+        "max_concurrent_shard_requests" : {
+          "type" : "number",
+          "description" : "The number of concurrent shard requests each sub search executes concurrently. This value should be used to limit the impact of the search on the cluster in order to limit the number of concurrent shard requests",
+          "default" : "The default grows with the number of nodes in the cluster but is at most 256."
         }
       }
     },

rest-api-spec/src/main/resources/rest-api-spec/test/msearch/10_basic.yml

@@ -61,3 +61,35 @@ setup:
   - match:  { responses.3.error.root_cause.0.reason: "/no.such.index/" }
   - match:  { responses.3.error.root_cause.0.index: index_3 }
   - match:  { responses.4.hits.total:     4  }
+
+---
+"Least impact smoke test":
+# only passing these parameters to make sure they are consumed
+  - do:
+      max_concurrent_shard_requests: 1
+      max_concurrent_searches: 1
+      msearch:
+        body:
+          - index: index_*
+          - query:
+              match: {foo: foo}
+          - index: index_2
+          - query:
+              match_all: {}
+          - index: index_1
+          - query:
+              match: {foo: foo}
+          - index: index_3
+          - query:
+              match_all: {}
+          - type: test
+          - query:
+              match_all: {}
+
+  - match:  { responses.0.hits.total:     2  }
+  - match:  { responses.1.hits.total:     1  }
+  - match:  { responses.2.hits.total:     1  }
+  - match:  { responses.3.error.root_cause.0.type: index_not_found_exception }
+  - match:  { responses.3.error.root_cause.0.reason: "/no.such.index/" }
+  - match:  { responses.3.error.root_cause.0.index: index_3 }
+  - match:  { responses.4.hits.total:     4  }

server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java

@@ -86,6 +86,14 @@ public class RestMultiSearchAction extends BaseRestHandler {
 
         int preFilterShardSize = restRequest.paramAsInt("pre_filter_shard_size", SearchRequest.DEFAULT_PRE_FILTER_SHARD_SIZE);
 
+        final Integer maxConcurrentShardRequests;
+        if (restRequest.hasParam("max_concurrent_shard_requests")) {
+            // only set if we have the parameter since we auto adjust the max concurrency on the coordinator
+            // based on the number of nodes in the cluster
+            maxConcurrentShardRequests = restRequest.paramAsInt("max_concurrent_shard_requests", Integer.MIN_VALUE);
+        } else {
+            maxConcurrentShardRequests = null;
+        }
 
         parseMultiLineRequest(restRequest, multiRequest.indicesOptions(), allowExplicitIndex, (searchRequest, parser) -> {
             searchRequest.source(SearchSourceBuilder.fromXContent(parser, false));
@@ -96,6 +104,9 @@ public class RestMultiSearchAction extends BaseRestHandler {
         for (SearchRequest request : requests) {
             // preserve if it's set on the request
             request.setPreFilterShardSize(Math.min(preFilterShardSize, request.getPreFilterShardSize()));
+            if (maxConcurrentShardRequests != null) {
+                request.setMaxConcurrentShardRequests(maxConcurrentShardRequests);
+            }
         }
         return multiRequest;
     }