Add _size and _doc_count to fields output (#70575)

Currently metadata fields like `_size` or `_doc_count` cannot be retrieved using
the fields API. With this change, we allow this if the field is explicitely
queried for using its name, but won't include metadata fields when e.g.
requesting all fields via "*".
With this change, not all metadata fields will be retrievable by using its name,
but support for "_size" and "_doc_count" (which is fetched from source) is
added. Support for other metadata field types will need to be decided case by
case and an appropriate ValueFetcher needs to be supplied.

Relates to #63569

docs/plugins/mapper-size.asciidoc

@@ -26,7 +26,7 @@ PUT my-index-000001
 --------------------------
 
 The value of the `_size` field is accessible in queries, aggregations, scripts,
-and when sorting:
+and when sorting. It can be retrieved using the {ref}/search-fields.html#search-fields-param[fields API]:
 
 [source,console]
 --------------------------
@@ -65,16 +65,12 @@ GET my-index-000001/_search
       }
     }
   ],
+  "fields": ["_size"],                <4>
   "script_fields": {
     "size": {
-      "script": "doc['_size']"        <4>
+      "script": "doc['_size']"        <5>
     }
-  },
-  "docvalue_fields": [
-    {
-      "field": "_size"                <5>
-    }
-  ]
+  }
 }
 --------------------------
 // TEST[continued]
@@ -82,12 +78,8 @@ GET my-index-000001/_search
 <1> Querying on the `_size` field
 <2> Aggregating on the `_size` field
 <3> Sorting on the `_size` field
-<4> Uses a
+<4> Use the `fields` parameter to return the `_size` in the search response.
+<5> Uses a
 {ref}/search-fields.html#script-fields[script field]
 to return the `_size` field in the search response.
-<5> Uses a
-{ref}/search-fields.html#docvalue-fields[doc value
-field] to return the `_size` field in the search response. Doc value fields are
-useful if
-{ref}/modules-scripting-security.html#allowed-script-types-setting[inline
-scripts are disabled].
+

plugins/mapper-size/src/internalClusterTest/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java

@@ -9,6 +9,7 @@ package org.elasticsearch.index.mapper.size;
 
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
 import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -93,11 +94,24 @@ public class SizeMappingIT extends ESIntegTestCase {
 
     public void testBasic() throws Exception {
         assertAcked(prepareCreate("test").setMapping("_size", "enabled=true"));
-        final String source = "{\"f\":10}";
+        final String source = "{\"f\":\"" + randomAlphaOfLengthBetween(1, 100)+ "\"}";
         indexRandom(true,
                 client().prepareIndex("test").setId("1").setSource(source, XContentType.JSON));
         GetResponse getResponse = client().prepareGet("test", "1").setStoredFields("_size").get();
         assertNotNull(getResponse.getField("_size"));
         assertEquals(source.length(), (int) getResponse.getField("_size").getValue());
     }
+
+    public void testGetWithFields() throws Exception {
+        assertAcked(prepareCreate("test").setMapping("_size", "enabled=true"));
+        final String source = "{\"f\":\"" + randomAlphaOfLengthBetween(1, 100)+ "\"}";
+        indexRandom(true,
+                client().prepareIndex("test").setId("1").setSource(source, XContentType.JSON));
+        SearchResponse searchResponse = client().prepareSearch("test").addFetchField("_size").get();
+        assertEquals(source.length(), ((Long) searchResponse.getHits().getHits()[0].getFields().get("_size").getValue()).intValue());
+
+        // this should not work when requesting fields via wildcard expression
+        searchResponse = client().prepareSearch("test").addFetchField("*").get();
+        assertNull(searchResponse.getHits().getHits()[0].getFields().get("_size"));
+    }
 }

plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java

@@ -9,12 +9,15 @@
 package org.elasticsearch.index.mapper.size;
 
 import org.elasticsearch.common.Explicit;
+import org.elasticsearch.index.mapper.DocValueFetcher;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
 import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType;
 import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType;
 import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.mapper.ValueFetcher;
+import org.elasticsearch.index.query.SearchExecutionContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -42,12 +45,26 @@ public class SizeFieldMapper extends MetadataFieldMapper {
 
         @Override
         public SizeFieldMapper build() {
-            return new SizeFieldMapper(enabled.getValue(), new NumberFieldType(NAME, NumberType.INTEGER));
+            return new SizeFieldMapper(enabled.getValue(), new SizeFieldType());
+        }
+    }
+
+    private static class SizeFieldType extends NumberFieldType {
+        SizeFieldType() {
+            super(NAME, NumberType.INTEGER);
+        }
+
+        @Override
+        public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
+            if (hasDocValues() == false) {
+                return lookup -> List.of();
+            }
+            return new DocValueFetcher(docValueFormat(format, null), context.getForField(this));
         }
     }
 
     public static final TypeParser PARSER = new ConfigurableTypeParser(
-        c -> new SizeFieldMapper(new Explicit<>(false, false), new NumberFieldType(NAME, NumberType.INTEGER)),
+        c -> new SizeFieldMapper(new Explicit<>(false, false), new SizeFieldType()),
         c -> new Builder()
     );
 

server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldFetcher.java

@@ -65,6 +65,7 @@ public class FieldFetcher {
 
         for (FieldAndFormat fieldAndFormat : fieldAndFormats) {
             String fieldPattern = fieldAndFormat.field;
+            boolean isWildcardPattern = Regex.isSimpleMatchPattern(fieldPattern);
             if (fieldAndFormat.includeUnmapped != null && fieldAndFormat.includeUnmapped) {
                 unmappedFetchPattern.add(fieldAndFormat.field);
             }
@@ -72,7 +73,11 @@ public class FieldFetcher {
             Collection<String> concreteFields = context.simpleMatchToIndexNames(fieldPattern);
             for (String field : concreteFields) {
                 MappedFieldType ft = context.getFieldType(field);
-                if (ft == null || context.isMetadataField(field)) {
+                if (ft == null) {
+                    continue;
+                }
+                // we want to skip metadata fields if we have a wildcard pattern
+                if (context.isMetadataField(field) && isWildcardPattern) {
                     continue;
                 }
                 if (field.startsWith(nestedScopePath) == false) {

server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java

@@ -182,15 +182,22 @@ public class FieldFetcherTests extends MapperServiceTestCase {
         MapperService mapperService = createMapperService();
         XContentBuilder source = XContentFactory.jsonBuilder().startObject()
             .field("field", "value")
+            .field("_doc_count", 100)
         .endObject();
 
-        Map<String, DocumentField> fields = fetchFields(mapperService, source, "_routing");
-        assertTrue(fields.isEmpty());
+        Map<String, DocumentField> fields = fetchFields(mapperService, source, "_doc_count");
+        assertNotNull(fields.get("_doc_count"));
+        assertEquals(100, ((Integer) fields.get("_doc_count").getValue()).intValue());
 
         // The _type field was deprecated in 7.x and is not supported in 8.0. So the behavior
         // should be the same as if the field didn't exist.
         fields = fetchFields(mapperService, source, "_type");
         assertTrue(fields.isEmpty());
+
+        // several other metadata fields throw exceptions via their value fetchers when trying to get them
+        for (String fieldname : List.of("_id", "_index", "_seq_no", "_routing", "_ignored")) {
+            expectThrows(UnsupportedOperationException.class, () -> fetchFields(mapperService, source, fieldname));
+        }
     }
 
     public void testFetchAllFields() throws IOException {