Add runtime field section to Field Capabilities API (#68904)

Currently runtime fields from search requests don't appear in the output of the
field capabilities API, but some consumer of runtime fields would like to see
runtime section just like they are defined in search requests reflected and
merged into the field capabilities output.
This change adds parsing of a "runtime_mappings" section equivallent to the one
on search requests to the `_field_caps` endpoint, passes this section down to
the shard level where any runtime fields defined here overwrite the mapping of
the targetet indices.

Closes #68117

docs/reference/search/field-caps.asciidoc

@@ -88,6 +88,13 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailab
 (Optional,  <<query-dsl,query object>> Allows to filter indices if the provided
 query rewrites to `match_none` on every shard.
 
+`runtime_mappings`::
+(Optional, object)
+Defines ad-hoc <<runtime-search-request,runtime fields>> in the request similar
+to the way it is done in <<search-api-body-runtime, search requests>>. These fields
+exist only as part of the query and take precedence over fields defined with the
+same name in the index mappings.
+
 [[search-field-caps-api-response-body]]
 ==== {api-response-body-title}
 

server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexRequest.java

@@ -20,6 +20,8 @@ import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.shard.ShardId;
 
 import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
 import java.util.Objects;
 
 public class FieldCapabilitiesIndexRequest extends ActionRequest implements IndicesRequest {
@@ -31,6 +33,7 @@ public class FieldCapabilitiesIndexRequest extends ActionRequest implements Indi
     private final OriginalIndices originalIndices;
     private final QueryBuilder indexFilter;
     private final long nowInMillis;
+    private Map<String, Object> runtimeFields;
 
     private ShardId shardId;
 
@@ -43,13 +46,15 @@ public class FieldCapabilitiesIndexRequest extends ActionRequest implements Indi
         originalIndices = OriginalIndices.readOriginalIndices(in);
         indexFilter = in.getVersion().onOrAfter(Version.V_7_9_0) ? in.readOptionalNamedWriteable(QueryBuilder.class) : null;
         nowInMillis =  in.getVersion().onOrAfter(Version.V_7_9_0) ? in.readLong() : 0L;
+        runtimeFields = in.getVersion().onOrAfter(Version.V_8_0_0) ? in.readMap() : Collections.emptyMap();
     }
 
     FieldCapabilitiesIndexRequest(String[] fields,
                                   String index,
                                   OriginalIndices originalIndices,
                                   QueryBuilder indexFilter,
-                                  long nowInMillis) {
+                                  long nowInMillis,
+                                  Map<String, Object> runtimeFields) {
         if (fields == null || fields.length == 0) {
             throw new IllegalArgumentException("specified fields can't be null or empty");
         }
@@ -58,6 +63,7 @@ public class FieldCapabilitiesIndexRequest extends ActionRequest implements Indi
         this.originalIndices = originalIndices;
         this.indexFilter = indexFilter;
         this.nowInMillis = nowInMillis;
+        this.runtimeFields = runtimeFields;
     }
 
     public String[] fields() {
@@ -82,6 +88,10 @@ public class FieldCapabilitiesIndexRequest extends ActionRequest implements Indi
         return indexFilter;
     }
 
+    public Map<String, Object> runtimeFields() {
+        return runtimeFields;
+    }
+
     public ShardId shardId() {
         return shardId;
     }
@@ -106,6 +116,9 @@ public class FieldCapabilitiesIndexRequest extends ActionRequest implements Indi
             out.writeOptionalNamedWriteable(indexFilter);
             out.writeLong(nowInMillis);
         }
+        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+            out.writeMap(runtimeFields);
+        }
     }
 
     @Override

server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java

@@ -23,7 +23,9 @@ import org.elasticsearch.index.query.QueryBuilder;
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashSet;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 
@@ -37,6 +39,7 @@ public final class FieldCapabilitiesRequest extends ActionRequest implements Ind
     // pkg private API mainly for cross cluster search to signal that we do multiple reductions ie. the results should not be merged
     private boolean mergeResults = true;
     private QueryBuilder indexFilter;
+    private Map<String, Object> runtimeFields = Collections.emptyMap();
     private Long nowInMillis;
 
     public FieldCapabilitiesRequest(StreamInput in) throws IOException {
@@ -52,6 +55,7 @@ public final class FieldCapabilitiesRequest extends ActionRequest implements Ind
         }
         indexFilter = in.getVersion().onOrAfter(Version.V_7_9_0) ? in.readOptionalNamedWriteable(QueryBuilder.class) : null;
         nowInMillis = in.getVersion().onOrAfter(Version.V_7_9_0) ? in.readOptionalLong() : null;
+        runtimeFields = in.getVersion().onOrAfter(Version.V_8_0_0) ? in.readMap() : Collections.emptyMap();
     }
 
     public FieldCapabilitiesRequest() {
@@ -90,6 +94,9 @@ public final class FieldCapabilitiesRequest extends ActionRequest implements Ind
             out.writeOptionalNamedWriteable(indexFilter);
             out.writeOptionalLong(nowInMillis);
         }
+        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+            out.writeMap(runtimeFields);
+        }
     }
 
     @Override
@@ -98,6 +105,9 @@ public final class FieldCapabilitiesRequest extends ActionRequest implements Ind
         if (indexFilter != null) {
             builder.field("index_filter", indexFilter);
         }
+        if (runtimeFields.isEmpty() == false) {
+            builder.field("runtime_mappings", runtimeFields);
+        }
         builder.endObject();
         return builder;
     }
@@ -121,6 +131,7 @@ public final class FieldCapabilitiesRequest extends ActionRequest implements Ind
     /**
      * The list of indices to lookup
      */
+    @Override
     public FieldCapabilitiesRequest indices(String... indices) {
         this.indices = Objects.requireNonNull(indices, "indices must not be null");
         return this;
@@ -166,6 +177,17 @@ public final class FieldCapabilitiesRequest extends ActionRequest implements Ind
     public QueryBuilder indexFilter() {
         return indexFilter;
     }
+    /**
+     * Allows adding search runtime fields if provided.
+     */
+    public FieldCapabilitiesRequest runtimeFields(Map<String, Object> runtimeFieldsSection) {
+        this.runtimeFields = runtimeFieldsSection;
+        return this;
+    }
+
+    public Map<String, Object> runtimeFields() {
+        return this.runtimeFields;
+    }
 
     Long nowInMillis() {
         return nowInMillis;
@@ -195,12 +217,13 @@ public final class FieldCapabilitiesRequest extends ActionRequest implements Ind
             indicesOptions.equals(that.indicesOptions) &&
             Arrays.equals(fields, that.fields) &&
             Objects.equals(indexFilter, that.indexFilter) &&
-            Objects.equals(nowInMillis, that.nowInMillis);
+            Objects.equals(nowInMillis, that.nowInMillis) &&
+            Objects.equals(runtimeFields, that.runtimeFields);
     }
 
     @Override
     public int hashCode() {
-        int result = Objects.hash(indicesOptions, includeUnmapped, mergeResults, indexFilter, nowInMillis);
+        int result = Objects.hash(indicesOptions, includeUnmapped, mergeResults, indexFilter, nowInMillis, runtimeFields);
         result = 31 * result + Arrays.hashCode(indices);
         result = 31 * result + Arrays.hashCode(fields);
         return result;

server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java

@@ -101,7 +101,7 @@ public class TransportFieldCapabilitiesAction extends HandledTransportAction<Fie
             };
             for (String index : concreteIndices) {
                 client.executeLocally(TransportFieldCapabilitiesIndexAction.TYPE, new FieldCapabilitiesIndexRequest(request.fields(),
-                    index, localIndices, request.indexFilter(), nowInMillis), innerListener);
+                    index, localIndices, request.indexFilter(), nowInMillis, request.runtimeFields()), innerListener);
             }
 
             // this is the cross cluster part of this API - we force the other cluster to not merge the results but instead

server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java

@@ -107,7 +107,7 @@ public class TransportFieldCapabilitiesIndexAction
         try (Engine.Searcher searcher = indexShard.acquireSearcher(Engine.CAN_MATCH_SEARCH_SOURCE)) {
 
             final SearchExecutionContext searchExecutionContext = indexService.newSearchExecutionContext(shardId.id(), 0,
-                searcher, request::nowInMillis, null, Collections.emptyMap());
+                searcher, request::nowInMillis, null, request.runtimeFields());
 
             if (canMatchShard(request, searchExecutionContext) == false) {
                 return new FieldCapabilitiesIndexResponse(request.index(), Collections.emptyMap(), false);

server/src/main/java/org/elasticsearch/rest/action/RestFieldCapabilitiesAction.java

@@ -11,13 +11,16 @@ package org.elasticsearch.rest.action;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 
 import java.io.IOException;
 import java.util.List;
 
+import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
@@ -50,9 +53,19 @@ public class RestFieldCapabilitiesAction extends BaseRestHandler {
         fieldRequest.includeUnmapped(request.paramAsBoolean("include_unmapped", false));
         request.withContentOrSourceParamParserOrNull(parser -> {
             if (parser != null) {
-                fieldRequest.indexFilter(RestActions.getQueryContent("index_filter", parser));
+                PARSER.parse(parser, fieldRequest, null);
             }
         });
         return channel -> client.fieldCaps(fieldRequest, new RestToXContentListener<>(channel));
     }
+
+    private static ParseField INDEX_FILTER_FIELD = new ParseField("index_filter");
+    private static ParseField RUNTIME_MAPPINGS_FIELD = new ParseField("runtime_mappings");
+
+    private static final ObjectParser<FieldCapabilitiesRequest, Void> PARSER = new ObjectParser<>("field_caps_request");
+
+    static {
+        PARSER.declareObject(FieldCapabilitiesRequest::indexFilter, (p, c) -> parseInnerQueryBuilder(p), INDEX_FILTER_FIELD);
+        PARSER.declareObject(FieldCapabilitiesRequest::runtimeFields, (p, c) -> p.map(), RUNTIME_MAPPINGS_FIELD);
+    }
 }

server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestTests.java

@@ -10,15 +10,27 @@ package org.elasticsearch.action.fieldcaps;
 
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.ArrayUtils;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.function.Consumer;
 
+import static java.util.Collections.singletonMap;
+
 public class FieldCapabilitiesRequestTests extends AbstractWireSerializingTestCase<FieldCapabilitiesRequest> {
 
     @Override
@@ -41,9 +53,24 @@ public class FieldCapabilitiesRequestTests extends AbstractWireSerializingTestCa
             request.indicesOptions(randomBoolean() ? IndicesOptions.strictExpand() : IndicesOptions.lenientExpandOpen());
         }
         request.includeUnmapped(randomBoolean());
+        if (randomBoolean()) {
+            request.nowInMillis(randomLong());
+        }
+        if (randomBoolean()) {
+            request.indexFilter(QueryBuilders.termQuery("field", randomAlphaOfLength(5)));
+        }
+        if (randomBoolean()) {
+            request.runtimeFields(Collections.singletonMap(randomAlphaOfLength(5), randomAlphaOfLength(5)));
+        }
         return request;
     }
 
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
+        return new NamedWriteableRegistry(searchModule.getNamedWriteables());
+    }
+
     @Override
     protected Writeable.Reader<FieldCapabilitiesRequest> instanceReader() {
         return FieldCapabilitiesRequest::new;
@@ -67,6 +94,11 @@ public class FieldCapabilitiesRequestTests extends AbstractWireSerializingTestCa
         });
         mutators.add(request -> request.setMergeResults(request.isMergeResults() == false));
         mutators.add(request -> request.includeUnmapped(request.includeUnmapped() == false));
+        mutators.add(request -> request.nowInMillis(request.nowInMillis() != null ? request.nowInMillis() + 1 : 1L));
+        mutators.add(
+            request -> request.indexFilter(request.indexFilter() != null ? request.indexFilter().boost(2) : QueryBuilders.matchAllQuery())
+        );
+        mutators.add(request -> request.runtimeFields(Collections.singletonMap("other_key", "other_value")));
 
         FieldCapabilitiesRequest mutatedInstance = copyInstance(instance);
         Consumer<FieldCapabilitiesRequest> mutator = randomFrom(mutators);
@@ -74,6 +106,32 @@ public class FieldCapabilitiesRequestTests extends AbstractWireSerializingTestCa
         return mutatedInstance;
     }
 
+    public void testToXContent() throws IOException {
+        FieldCapabilitiesRequest request = new FieldCapabilitiesRequest();
+        request.indexFilter(QueryBuilders.termQuery("field", "value"));
+        request.runtimeFields(singletonMap("day_of_week", singletonMap("type", "keyword")));
+        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+        String xContent = BytesReference.bytes(request.toXContent(builder, ToXContent.EMPTY_PARAMS)).utf8ToString();
+        assertEquals(
+            ("{"
+                + "  \"index_filter\": {\n"
+                + "    \"term\": {\n"
+                + "      \"field\": {\n"
+                + "        \"value\": \"value\",\n"
+                + "        \"boost\": 1.0\n"
+                + "      }\n"
+                + "    }\n"
+                + "  },\n"
+                + "  \"runtime_mappings\": {\n"
+                + "    \"day_of_week\": {\n"
+                + "      \"type\": \"keyword\"\n"
+                + "    }\n"
+                + "  }\n"
+                + "}").replaceAll("\\s+", ""),
+            xContent
+        );
+    }
+
     public void testValidation() {
         FieldCapabilitiesRequest request = new FieldCapabilitiesRequest()
             .indices("index2");

x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/40_runtime_mappings.yml

@@ -0,0 +1,87 @@
+---
+setup:
+  - do:
+        indices.create:
+          index: test-1
+          body:
+            mappings:
+              properties:
+                timestamp:
+                  type: date
+
+  - do:
+      index:
+        index:  test-1
+        body:   { timestamp: "2015-01-02" }
+
+  - do:
+      indices.refresh:
+        index: [test-1]
+
+---
+"Field caps with runtime mappings section":
+
+  - skip:
+      version: " - 7.99.99"
+      reason: Runtime mappings support was added in 8.0
+
+  - do:
+      field_caps:
+        index: test-*
+        fields: "*"
+        body:
+          runtime_mappings:
+            day_of_week:
+              type: keyword
+              script:
+                source: "emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
+
+  - match: {indices:                             ["test-1"]}
+  - length: {fields.timestamp:                            1}
+  - match: {fields.timestamp.date.type:                date}
+  - match: {fields.timestamp.date.searchable:          true}
+  - match: {fields.timestamp.date.aggregatable:        true}
+  - length: {fields.day_of_week:                          1}
+  - match: {fields.day_of_week.keyword.type:           keyword}
+  - match: {fields.day_of_week.keyword.searchable:        true}
+  - match: {fields.day_of_week.keyword.aggregatable:      true}
+
+---
+"Field caps with runtime mappings section overwriting existing mapping":
+
+  - skip:
+      version: " - 7.99.99"
+      reason: Runtime mappings support was added in 8.0
+
+  - do:
+      index:
+        index:  test-2
+        body:   { day_of_week: 123 }
+
+  - do:
+      field_caps:
+        index: test-*
+        fields: "day*"
+
+  - match: {indices:                    ["test-1", "test-2"]}
+  - length: {fields.day_of_week:                          1}
+  - match: {fields.day_of_week.long.type:          long}
+  - match: {fields.day_of_week.long.searchable:        true}
+  - match: {fields.day_of_week.long.aggregatable:      true}
+
+  - do:
+      field_caps:
+        index: test-*
+        fields: "day*"
+        body:
+          runtime_mappings:
+            day_of_week:
+              type: keyword
+              script:
+                source: "emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
+
+  - match: {indices:                    ["test-1", "test-2"]}
+  - length: {fields.day_of_week:                          1}
+  - match: {fields.day_of_week.keyword.type:          keyword}
+  - match: {fields.day_of_week.keyword.searchable:        true}
+  - match: {fields.day_of_week.keyword.aggregatable:      true}