Add `_index` and `_version` metatada fields to fields api (#79042)

Currently we don't allow retrieving metadata fields through the fields option in search but throw
an error on this case. In #78828 we started to enable this for "_id" if the field is explicitely requested.
This PR adds _index and _version metadata fields which are internally stored as doc values to
the list of fields that can be explicitely retrieved.

Relates to #75836

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml

@@ -1058,7 +1058,9 @@ test fetching metadata fields:
       search:
         index: test
         body:
-          fields: [ "_id" ]
+          fields: [ "_id", "_index", "_version" ]
 
-  - length: { hits.hits.0.fields : 1 }
+  - length: { hits.hits.0.fields : 3 }
   - match: { hits.hits.0.fields._id.0: "1" }
+  - match: { hits.hits.0.fields._index.0: "test" }
+  - match: { hits.hits.0.fields._version.0: 1 }

server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java

@@ -16,8 +16,11 @@ import org.elasticsearch.index.fielddata.plain.ConstantIndexFieldData;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
 import org.elasticsearch.search.lookup.SearchLookup;
+import org.elasticsearch.search.lookup.SourceLookup;
 
+import java.io.IOException;
 import java.util.Collections;
+import java.util.List;
 import java.util.function.Supplier;
 
 public class IndexFieldMapper extends MetadataFieldMapper {
@@ -65,7 +68,15 @@ public class IndexFieldMapper extends MetadataFieldMapper {
 
         @Override
         public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
-            throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "].");
+            return new ValueFetcher() {
+
+                private final List<Object> indexName = List.of(context.getFullyQualifiedIndex().getName());
+
+                @Override
+                public List<Object> fetchValues(SourceLookup lookup, List<Object> ignoredValues) throws IOException {
+                    return indexName;
+                }
+            };
         }
     }
 

server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java

@@ -11,10 +11,15 @@ package org.elasticsearch.index.mapper;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.search.Query;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
+import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData;
 import org.elasticsearch.index.query.QueryShardException;
 import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.util.Collections;
+import java.util.function.Supplier;
 
 /** Mapper for the _version field. */
 public class VersionFieldMapper extends MetadataFieldMapper {
@@ -46,7 +51,13 @@ public class VersionFieldMapper extends MetadataFieldMapper {
 
         @Override
         public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
-            throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "].");
+            return new DocValueFetcher(docValueFormat(format, null), context.getForField(this));
+        }
+
+        @Override
+        public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier<SearchLookup> searchLookup) {
+            failIfNoDocValues();
+            return new SortedNumericIndexFieldData.Builder(name(), NumericType.LONG);
         }
     }
 

server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java

@@ -8,9 +8,23 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.IndexSearcher;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.fielddata.IndexFieldDataCache;
+import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
+import org.elasticsearch.search.lookup.SearchLookup;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Mockito.when;
 
 public class IndexFieldMapperTests extends MapperServiceTestCase {
 
@@ -27,4 +41,29 @@ public class IndexFieldMapperTests extends MapperServiceTestCase {
         assertThat(e.getMessage(), containsString("_index is not configurable"));
     }
 
+    public void testFetchFieldValue() throws IOException {
+        MapperService mapperService = createMapperService(
+            fieldMapping(b -> b.field("type", "keyword"))
+        );
+        String index = randomAlphaOfLength(12);
+        withLuceneIndex(mapperService, iw -> {
+            SourceToParse source = source(index, "id", b -> b.field("field", "value"), "", Map.of());
+            iw.addDocument(mapperService.documentMapper().parse(source).rootDoc());
+        }, iw -> {
+            IndexFieldMapper.IndexFieldType ft = (IndexFieldMapper.IndexFieldType) mapperService.fieldType("_index");
+            SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup());
+            SearchExecutionContext searchExecutionContext = createSearchExecutionContext(mapperService);
+            when(searchExecutionContext.getForField(ft)).thenReturn(
+                ft.fielddataBuilder(index, () -> lookup).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService())
+            );
+            when(searchExecutionContext.getFullyQualifiedIndex()).thenReturn(new Index(index, "indexUUid"));
+            ValueFetcher valueFetcher = ft.valueFetcher(searchExecutionContext, null);
+            IndexSearcher searcher = newSearcher(iw);
+            LeafReaderContext context = searcher.getIndexReader().leaves().get(0);
+            lookup.source().setSegmentAndDocument(context, 0);
+            valueFetcher.setNextReader(context);
+            assertEquals(List.of(index), valueFetcher.fetchValues(lookup.source(), Collections.emptyList()));
+        });
+    }
+
 }

server/src/test/java/org/elasticsearch/index/mapper/VersionFieldMapperTests.java

@@ -0,0 +1,76 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.IndexSearcher;
+import org.elasticsearch.index.fielddata.IndexFieldDataCache;
+import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
+import org.elasticsearch.search.lookup.SearchLookup;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.mockito.Mockito.when;
+
+public class VersionFieldMapperTests extends MapperServiceTestCase {
+
+    public void testIncludeInObjectNotAllowed() throws Exception {
+        DocumentMapper docMapper = createDocumentMapper(mapping(b -> {}));
+
+        Exception e = expectThrows(MapperParsingException.class,
+            () -> docMapper.parse(source(b -> b.field("_version", 1))));
+
+        assertThat(e.getCause().getMessage(),
+            containsString("Field [_version] is a metadata field and cannot be added inside a document"));
+    }
+
+    public void testDefaults() throws IOException {
+        DocumentMapper mapper = createDocumentMapper(mapping(b -> {}));
+        ParsedDocument document = mapper.parse(source(b -> b.field("field", "value")));
+        IndexableField[] fields = document.rootDoc().getFields(VersionFieldMapper.NAME);
+        assertEquals(1, fields.length);
+        assertEquals(IndexOptions.NONE, fields[0].fieldType().indexOptions());
+        assertEquals(DocValuesType.NUMERIC, fields[0].fieldType().docValuesType());
+    }
+
+    public void testFetchFieldValue() throws IOException {
+        MapperService mapperService = createMapperService(
+            fieldMapping(b -> b.field("type", "keyword"))
+        );
+        long version = randomLongBetween(1, 1000);
+        withLuceneIndex(mapperService, iw -> {
+            ParsedDocument parsedDoc = mapperService.documentMapper().parse(source(b -> b.field("field", "value")));
+            parsedDoc.version().setLongValue(version);
+            iw.addDocument(parsedDoc.rootDoc());
+        }, iw -> {
+            VersionFieldMapper.VersionFieldType ft = (VersionFieldMapper.VersionFieldType) mapperService.fieldType("_version");
+            SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup());
+            SearchExecutionContext searchExecutionContext = createSearchExecutionContext(mapperService);
+            when(searchExecutionContext.getForField(ft)).thenReturn(
+                ft.fielddataBuilder("test", () -> lookup).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService())
+            );
+            ValueFetcher valueFetcher = ft.valueFetcher(searchExecutionContext, null);
+            IndexSearcher searcher = newSearcher(iw);
+            LeafReaderContext context = searcher.getIndexReader().leaves().get(0);
+            lookup.source().setSegmentAndDocument(context, 0);
+            valueFetcher.setNextReader(context);
+            assertEquals(List.of(version), valueFetcher.fetchValues(lookup.source(), Collections.emptyList()));
+        });
+    }
+
+
+
+}

server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java

@@ -20,10 +20,14 @@ import org.elasticsearch.common.document.DocumentField;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.MapperServiceTestCase;
+import org.elasticsearch.index.mapper.NestedPathFieldMapper;
 import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.index.mapper.SeqNoFieldMapper;
+import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.search.lookup.SourceLookup;
@@ -201,18 +205,19 @@ public class FieldFetcherTests extends MapperServiceTestCase {
         fields = fetchFields(mapperService, source, "_type");
         assertTrue(fields.isEmpty());
 
-        // several other metadata fields throw exceptions via their value fetchers when trying to get them
-        for (String fieldname : List.of("_index", "_seq_no")) {
-            expectThrows(UnsupportedOperationException.class, () -> fetchFields(mapperService, source, fieldname));
-        }
-
         String docId = randomAlphaOfLength(12);
         String routing = randomAlphaOfLength(12);
+        long version = randomLongBetween(1, 100);
         withLuceneIndex(mapperService, iw -> {
-            iw.addDocument(mapperService.documentMapper().parse(source(docId, b -> b.field("integer_field", "value"), routing)).rootDoc());
+            ParsedDocument parsedDocument = mapperService.documentMapper()
+                .parse(source(docId, b -> b.field("integer_field", "value"), routing));
+            parsedDocument.version().setLongValue(version);
+            iw.addDocument(parsedDocument.rootDoc());
         }, iw -> {
             List<FieldAndFormat> fieldList = List.of(
                 new FieldAndFormat("_id", null),
+                new FieldAndFormat("_index", null),
+                new FieldAndFormat("_version", null),
                 new FieldAndFormat("_routing", null),
                 new FieldAndFormat("_ignored", null)
             );
@@ -228,11 +233,23 @@ public class FieldFetcherTests extends MapperServiceTestCase {
             sourceLookup.setSegmentAndDocument(readerContext, 0);
 
             Map<String, DocumentField> fetchedFields = fieldFetcher.fetch(sourceLookup);
-            assertThat(fetchedFields.size(), equalTo(3));
+            assertThat(fetchedFields.size(), equalTo(5));
             assertEquals(docId, fetchedFields.get("_id").getValue());
             assertEquals(routing, fetchedFields.get("_routing").getValue());
+            assertEquals("test", fetchedFields.get("_index").getValue());
+            assertEquals(version, ((Long) fetchedFields.get("_version").getValue()).longValue());
             assertEquals("integer_field", fetchedFields.get("_ignored").getValue());
         });
+
+        // several other metadata fields throw exceptions via their value fetchers when trying to get them
+        for (String fieldname : List.of(
+            SeqNoFieldMapper.NAME,
+            SourceFieldMapper.NAME,
+            FieldNamesFieldMapper.NAME,
+            NestedPathFieldMapper.name(Version.CURRENT)
+        )) {
+            expectThrows(UnsupportedOperationException.class, () -> fetchFields(mapperService, source, fieldname));
+        }
     }
 
     public void testFetchAllFields() throws IOException {
@@ -1016,7 +1033,7 @@ public class FieldFetcherTests extends MapperServiceTestCase {
             .put("index.number_of_shards", 1)
             .put("index.number_of_replicas", 0)
             .put(IndexMetadata.SETTING_INDEX_UUID, "uuid").build();
-        IndexMetadata indexMetadata = new IndexMetadata.Builder("index").settings(settings).build();
+        IndexMetadata indexMetadata = new IndexMetadata.Builder("test").settings(settings).build();
         IndexSettings indexSettings = new IndexSettings(indexMetadata, settings);
         return new SearchExecutionContext(
             0,

test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java

@@ -235,13 +235,15 @@ public abstract class MapperServiceTestCase extends ESTestCase {
 
     protected final SourceToParse source(String id, CheckedConsumer<XContentBuilder, IOException> build, @Nullable String routing)
         throws IOException {
-        XContentBuilder builder = JsonXContent.contentBuilder().startObject();
-        build.accept(builder);
-        builder.endObject();
-        return new SourceToParse("test", id, BytesReference.bytes(builder), XContentType.JSON, routing, Map.of());
+        return source("test", id, build, routing, Map.of());
     }
 
     protected final SourceToParse source(String id, CheckedConsumer<XContentBuilder, IOException> build,
+        @Nullable String routing, Map<String, String> dynamicTemplates) throws IOException {
+        return source("text", id, build, routing, dynamicTemplates);
+    }
+
+    protected final SourceToParse source(String index, String id, CheckedConsumer<XContentBuilder, IOException> build,
                                          @Nullable String routing, Map<String, String> dynamicTemplates) throws IOException {
         XContentBuilder builder = JsonXContent.contentBuilder().startObject();
         build.accept(builder);