Search - return ignored field values from fields api. (#78697)

Since Kibana's Discover switched to retrieving values via the fields API rather than source there have been gaps in the display caused by "ignored" fields (those that fall foul of ignore_above and ignore_malformed size and formatting rules).

This PR returns ignored values from source when a user-requested field fails to be parsed for a document. In these cases the corresponding hit adds a new ignored_field_values section in the response.

Closes #74121

docs/reference/search/search-your-data/retrieve-selected-fields.asciidoc

@@ -376,6 +376,96 @@ won't be included in the response because `include_unmapped` isn't set to
 // TESTRESPONSE[s/"max_score" : 1.0/"max_score" : $body.hits.max_score/]
 // TESTRESPONSE[s/"_score" : 1.0/"_score" : $body.hits.hits.0._score/]
 
+[discrete]
+[[Ignored-field values]]
+==== Ignored field values
+The `fields` option only returns values that were valid when indexed.
+If your search request asks for values from a field that ignored certain
+because they were malformed or too large these values are returned
+separately in an `ignored_field_values` section.
+
+In this example we index a document that has a value which is ignored and
+not added to the index so is shown separately in search results:
+
+[source,console]
+----
+PUT my-index-000001
+{
+  "mappings": {
+    "properties": {
+      "my-small" : { "type" : "keyword", "ignore_above": 2 }, <1>
+      "my-large" : { "type" : "keyword" }
+    }
+  }
+}
+
+PUT my-index-000001/_doc/1?refresh=true
+{
+  "my-small": ["ok", "bad"], <2>
+  "my-large": "ok content"
+}
+
+POST my-index-000001/_search
+{
+  "fields": ["my-*"],
+  "_source": false
+}
+----
+
+<1> This field has a size restriction
+<2> This document field has a value that exceeds the size restriction so is ignored and not indexed
+
+The response will contain ignored field values under the  `ignored_field_values` path.
+These values are retrieved from the document's original JSON source and are raw so will
+not be formatted or treated in any way, unlike the successfully indexed fields which are
+returned in the `fields` section.
+
+[source,console-result]
+----
+{
+  "took" : 2,
+  "timed_out" : false,
+  "_shards" : {
+    "total" : 1,
+    "successful" : 1,
+    "skipped" : 0,
+    "failed" : 0
+  },
+  "hits" : {
+    "total" : {
+      "value" : 1,
+      "relation" : "eq"
+    },
+    "max_score" : 1.0,
+    "hits" : [
+      {
+        "_index" : "my-index-000001",
+        "_id" : "1",
+        "_score" : 1.0,
+        "_ignored" : [ "my-small"],
+        "fields" : {
+          "my-large": [
+            "ok content"
+          ],
+          "my-small": [
+            "ok"
+          ]
+        },
+        "ignored_field_values" : {
+          "my-small": [
+            "bad"
+          ]
+        }
+      }
+    ]
+  }
+}
+----
+// TESTRESPONSE[s/"took" : 2/"took": $body.took/]
+// TESTRESPONSE[s/"max_score" : 1.0/"max_score" : $body.hits.max_score/]
+// TESTRESPONSE[s/"_score" : 1.0/"_score" : $body.hits.hits.0._score/]
+
+
 [discrete]
 [[source-filtering]]
 === The `_source` option

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java

@@ -48,6 +48,7 @@ import org.elasticsearch.search.lookup.SourceLookup;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -178,7 +179,7 @@ public class MatchOnlyTextFieldMapper extends FieldMapper {
                 return docID -> {
                     try {
                         sourceLookup.setSegmentAndDocument(context, docID);
-                        return valueFetcher.fetchValues(sourceLookup);
+                        return valueFetcher.fetchValues(sourceLookup, new ArrayList<>());
                     } catch (IOException e) {
                         throw new UncheckedIOException(e);
                     }

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java

@@ -109,7 +109,7 @@ public class TokenCountFieldMapper extends FieldMapper {
         @Override
         public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
             if (hasDocValues() == false) {
-                return lookup -> List.of();
+                return (lookup, ignoredValues) -> List.of();
             }
             return new DocValueFetcher(docValueFormat(format, null), context.getForField(this));
         }

modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java

@@ -76,7 +76,7 @@ public final class ParentIdFieldMapper extends FieldMapper {
         public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
             // Although this is an internal field, we return it in the list of all field types. So we
             // provide an empty value fetcher here instead of throwing an error.
-            return lookup -> List.of();
+            return (lookup, ignoredValues) -> List.of();
         }
 
         @Override

plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java

@@ -57,7 +57,7 @@ public class SizeFieldMapper extends MetadataFieldMapper {
         @Override
         public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
             if (hasDocValues() == false) {
-                return lookup -> List.of();
+                return (lookup, ignoredValues) -> List.of();
             }
             return new DocValueFetcher(docValueFormat(format, null), context.getForField(this));
         }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/200_ignore_malformed.yml

@@ -88,3 +88,20 @@ setup:
 
   - length:   { hits.hits: 1  }
   - is_true:  hits.hits.0._ignored
+
+
+---
+"ignored_field_values is returned by default":
+  - skip:
+      version: " - 7.99.99"
+      reason: ignored_field_values was added in 8.0
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body: { query: { ids: { "values": [ "3" ] } },"_source":false, "fields":["my*"]}
+
+  - length:   { hits.hits: 1  }
+  - length:   { hits.hits.0._ignored: 2}
+  - length:   { hits.hits.0.ignored_field_values.my_date: 1}
+  - length:   { hits.hits.0.ignored_field_values.my_ip: 1}

server/src/main/java/org/elasticsearch/common/document/DocumentField.java

@@ -8,6 +8,7 @@
 
 package org.elasticsearch.common.document;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -19,6 +20,7 @@ import org.elasticsearch.search.SearchHit;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Objects;
@@ -32,21 +34,33 @@ import static org.elasticsearch.common.xcontent.XContentParserUtils.parseFieldsV
  * @see SearchHit
  * @see GetResult
  */
-public class DocumentField implements Writeable, ToXContentFragment, Iterable<Object> {
+public class DocumentField implements Writeable, Iterable<Object> {
 
     private final String name;
     private final List<Object> values;
-
+    private List<Object> ignoredValues;
+    
     public DocumentField(StreamInput in) throws IOException {
         name = in.readString();
         values = in.readList(StreamInput::readGenericValue);
+        if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+            ignoredValues = in.readList(StreamInput::readGenericValue);
+        } else {
+            ignoredValues = Collections.emptyList();
+        }
     }
 
     public DocumentField(String name, List<Object> values) {
+        this(name, values, Collections.emptyList());
+    }
+
+    public DocumentField(String name, List<Object> values, List<Object> ignoredValues) {
         this.name = Objects.requireNonNull(name, "name must not be null");
         this.values = Objects.requireNonNull(values, "values must not be null");
+        this.ignoredValues = Objects.requireNonNull(ignoredValues, "ignoredValues must not be null");
     }
 
+    
     /**
      * The name of the field.
      */
@@ -76,25 +90,55 @@ public class DocumentField implements Writeable, ToXContentFragment, Iterable<Ob
     public Iterator<Object> iterator() {
         return values.iterator();
     }
+    
+    /**
+     * The field's ignored values as an immutable list.
+     */
+    public List<Object> getIgnoredValues() {
+        return ignoredValues == Collections.emptyList() ? ignoredValues : Collections.unmodifiableList(ignoredValues);
+    }    
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(name);
         out.writeCollection(values, StreamOutput::writeGenericValue);
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startArray(name);
-        for (Object value : values) {
-            // This call doesn't really need to support writing any kind of object, since the values
-            // here are always serializable to xContent. Each value could be a leaf types like a string,
-            // number, or boolean, a list of such values, or a map of such values with string keys.
-            builder.value(value);
+        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+            out.writeCollection(ignoredValues, StreamOutput::writeGenericValue);
         }
-        builder.endArray();
-        return builder;
+        
     }
+    
+    public ToXContentFragment getValidValuesWriter(){
+        return new ToXContentFragment() {
+            
+            @Override
+            public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+                builder.startArray(name);
+                for (Object value : values) {
+                    // This call doesn't really need to support writing any kind of object, since the values
+                    // here are always serializable to xContent. Each value could be a leaf types like a string,
+                    // number, or boolean, a list of such values, or a map of such values with string keys.
+                    builder.value(value);
+                }
+                builder.endArray();
+                return builder;
+            }
+        };
+    }
+    public ToXContentFragment getIgnoredValuesWriter(){
+        return new ToXContentFragment() {
+            
+            @Override
+            public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+                builder.startArray(name);
+                for (Object value : ignoredValues) {
+                    builder.value(value);
+                }
+                builder.endArray();
+                return builder;
+            }
+        };
+    }    
 
     public static DocumentField fromXContent(XContentParser parser) throws IOException {
         ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser);
@@ -117,12 +161,13 @@ public class DocumentField implements Writeable, ToXContentFragment, Iterable<Ob
             return false;
         }
         DocumentField objects = (DocumentField) o;
-        return Objects.equals(name, objects.name) && Objects.equals(values, objects.values);
+        return Objects.equals(name, objects.name) && Objects.equals(values, objects.values) && 
+            Objects.equals(ignoredValues, objects.ignoredValues);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(name, values);
+        return Objects.hash(name, values, ignoredValues);
     }
 
     @Override
@@ -130,6 +175,8 @@ public class DocumentField implements Writeable, ToXContentFragment, Iterable<Ob
         return "DocumentField{" +
                 "name='" + name + '\'' +
                 ", values=" + values +
+                ", ignoredValues=" + ignoredValues +
                 '}';
     }
+
 }

server/src/main/java/org/elasticsearch/index/get/GetResult.java

@@ -275,7 +275,7 @@ public class GetResult implements Writeable, Iterable<DocumentField>, ToXContent
         if (documentFields.isEmpty() == false) {
             builder.startObject(FIELDS);
             for (DocumentField field : documentFields.values()) {
-                field.toXContent(builder, params);
+                field.getValidValuesWriter().toXContent(builder, params);
             }
             builder.endObject();
         }

server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java

@@ -43,7 +43,7 @@ public abstract class ArraySourceValueFetcher implements ValueFetcher {
     }
 
     @Override
-    public List<Object> fetchValues(SourceLookup lookup) {
+    public List<Object> fetchValues(SourceLookup lookup, List<Object> ignoredValues) {
         List<Object> values = new ArrayList<>();
         for (String path : sourcePaths) {
             Object sourceValue = lookup.extractValue(path, nullValue);
@@ -55,7 +55,7 @@ public abstract class ArraySourceValueFetcher implements ValueFetcher {
             } catch (Exception e) {
                 // if parsing fails here then it would have failed at index time
                 // as well, meaning that we must be ignoring malformed values.
-                // So ignore it here too.
+                ignoredValues.add(sourceValue);
             }
         }
         return values;

server/src/main/java/org/elasticsearch/index/mapper/DocValueFetcher.java

@@ -39,7 +39,7 @@ public final class DocValueFetcher implements ValueFetcher {
     }
 
     @Override
-    public List<Object> fetchValues(SourceLookup lookup) throws IOException {
+    public List<Object> fetchValues(SourceLookup lookup, List<Object> ignoredValues) throws IOException {
         if (false == formattedDocValues.advanceExact(lookup.docId())) {
             return emptyList();
         }

server/src/main/java/org/elasticsearch/index/mapper/NestedValueFetcher.java

@@ -39,7 +39,7 @@ public class NestedValueFetcher implements ValueFetcher {
     }
 
     @Override
-    public List<Object> fetchValues(SourceLookup lookup) throws IOException {
+    public List<Object> fetchValues(SourceLookup lookup, List<Object> includedValues) throws IOException {
         List<Object> nestedEntriesToReturn = new ArrayList<>();
         Map<String, Object> filteredSource = new HashMap<>();
         Map<String, Object> stub = createSourceMapStub(filteredSource);

server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java

@@ -51,7 +51,7 @@ public abstract class SourceValueFetcher implements ValueFetcher {
     }
 
     @Override
-    public List<Object> fetchValues(SourceLookup lookup) {
+    public List<Object> fetchValues(SourceLookup lookup, List<Object> ignoredValues) {
         List<Object> values = new ArrayList<>();
         for (String path : sourcePaths) {
             Object sourceValue = lookup.extractValue(path, nullValue);
@@ -76,8 +76,11 @@ public abstract class SourceValueFetcher implements ValueFetcher {
                         Object parsedValue = parseSourceValue(value);
                         if (parsedValue != null) {
                             values.add(parsedValue);
+                        } else {
+                            ignoredValues.add(value);
                         }
                     } catch (Exception e) {
+                        ignoredValues.add(value);
                         // if we get a parsing exception here, that means that the
                         // value in _source would have also caused a parsing
                         // exception at index time and the value ignored.

server/src/main/java/org/elasticsearch/index/mapper/StoredValueFetcher.java

@@ -36,7 +36,7 @@ public final class StoredValueFetcher implements ValueFetcher {
     }
 
     @Override
-    public List<Object> fetchValues(SourceLookup lookup) throws IOException {
+    public List<Object> fetchValues(SourceLookup lookup, List<Object> ignoredValues) throws IOException {
         leafSearchLookup.setDocument(lookup.docId());
         return leafSearchLookup.fields().get(fieldname).getValues();
     }

server/src/main/java/org/elasticsearch/index/mapper/ValueFetcher.java

@@ -31,9 +31,10 @@ public interface ValueFetcher {
     * should not be relied on.
     *
     * @param lookup a lookup structure over the document's source.
+    * @param ignoredValues a mutable list to collect any ignored values as they were originally presented in source
     * @return a list a standardized field values.
     */
-    List<Object> fetchValues(SourceLookup lookup) throws IOException;
+    List<Object> fetchValues(SourceLookup lookup, List<Object> ignoredValues) throws IOException;
 
     /**
      * Update the leaf reader used to fetch values.

server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java

@@ -256,7 +256,8 @@ public class TermVectorsService  {
             for (String field : fields) {
                 if (values.containsKey(field) == false) {
                     SourceValueFetcher valueFetcher = SourceValueFetcher.toString(mappingLookup.sourcePaths(field));
-                    List<Object> v = valueFetcher.fetchValues(sourceLookup);
+                    List<Object> ignoredValues = new ArrayList<>();
+                    List<Object> v = valueFetcher.fetchValues(sourceLookup, ignoredValues);
                     if (v.isEmpty() == false) {
                         values.put(field, v);
                     }

server/src/main/java/org/elasticsearch/search/SearchHit.java

@@ -582,6 +582,7 @@ public final class SearchHit implements Writeable, ToXContentObject, Iterable<Do
         static final String _PRIMARY_TERM = "_primary_term";
         static final String _SCORE = "_score";
         static final String FIELDS = "fields";
+        static final String IGNORED_FIELD_VALUES = "ignored_field_values";
         static final String HIGHLIGHT = "highlight";
         static final String SORT = "sort";
         static final String MATCHED_QUERIES = "matched_queries";
@@ -666,7 +667,19 @@ public final class SearchHit implements Writeable, ToXContentObject, Iterable<Do
             builder.startObject(Fields.FIELDS);
             for (DocumentField field : documentFields.values()) {
                 if (field.getValues().size() > 0) {
-                    field.toXContent(builder, params);
+                    field.getValidValuesWriter().toXContent(builder, params);
+                }
+            }
+            builder.endObject();
+        }
+        //ignored field values
+        if (documentFields.isEmpty() == false &&
+            // omit ignored_field_values all together if there are none
+            documentFields.values().stream().anyMatch(df -> df.getIgnoredValues().size() > 0)) {
+            builder.startObject(Fields.IGNORED_FIELD_VALUES);
+            for (DocumentField field : documentFields.values()) {
+                if (field.getIgnoredValues().size() > 0) {
+                    field.getIgnoredValuesWriter().toXContent(builder, params);
                 }
             }
             builder.endObject();

server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java

@@ -69,7 +69,10 @@ public final class FetchDocValuesPhase implements FetchSubPhase {
                         // docValues fields will still be document fields, and put under "fields" section of a hit.
                         hit.hit().setDocumentField(f.field, hitField);
                     }
-                    hitField.getValues().addAll(f.fetcher.fetchValues(hit.sourceLookup()));
+                    List <Object> ignoredValues = new ArrayList<>();
+                    hitField.getValues().addAll(f.fetcher.fetchValues(hit.sourceLookup(), ignoredValues));
+                    // Doc value fetches should not return any ignored values
+                    assert ignoredValues.isEmpty();
                 }
             }
         };

server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldFetcher.java

@@ -167,9 +167,10 @@ public class FieldFetcher {
             String field = context.fieldName;
 
             ValueFetcher valueFetcher = context.valueFetcher;
-            List<Object> parsedValues = valueFetcher.fetchValues(sourceLookup);
-            if (parsedValues.isEmpty() == false) {
-                documentFields.put(field, new DocumentField(field, parsedValues));
+            List<Object> ignoredValues = new ArrayList<>();
+            List<Object> parsedValues = valueFetcher.fetchValues(sourceLookup, ignoredValues);
+            if (parsedValues.isEmpty() == false || ignoredValues.isEmpty() == false) {
+                documentFields.put(field, new DocumentField(field, parsedValues, ignoredValues));
             }
         }
         collectUnmapped(documentFields, sourceLookup.source(), "", 0);

server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightUtils.java

@@ -17,6 +17,7 @@ import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.search.fetch.FetchSubPhase;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
@@ -48,7 +49,7 @@ public final class HighlightUtils {
         }
         ValueFetcher fetcher = fieldType.valueFetcher(searchContext, null);
         fetcher.setNextReader(hitContext.readerContext());
-        return fetcher.fetchValues(hitContext.sourceLookup());
+        return fetcher.fetchValues(hitContext.sourceLookup(), new ArrayList<Object>());
     }
 
     public static class Encoders {

server/src/main/java/org/elasticsearch/search/lookup/FieldValues.java

@@ -58,13 +58,12 @@ public interface FieldValues<T> {
             }
 
             @Override
-            public List<Object> fetchValues(SourceLookup lookup)  {
+            public List<Object> fetchValues(SourceLookup lookup, List<Object> ignoredValues)  {
                 List<Object> values = new ArrayList<>();
                 try {
                     fieldValues.valuesForDoc(context.lookup(), ctx, lookup.docId(), v -> values.add(formatter.apply(v)));
                 } catch (Exception e) {
-                    // ignore errors - if they exist here then they existed at index time
-                    // and so on_script_error must have been set to `ignore`
+                    ignoredValues.addAll(values);
                 }
                 return values;
             }
@@ -89,13 +88,12 @@ public interface FieldValues<T> {
             }
 
             @Override
-            public List<Object> fetchValues(SourceLookup lookup)  {
+            public List<Object> fetchValues(SourceLookup lookup, List<Object> ignoredValues)  {
                 List<T> values = new ArrayList<>();
                 try {
                     fieldValues.valuesForDoc(context.lookup(), ctx, lookup.docId(), v -> values.add(v));
                 } catch (Exception e) {
-                    // ignore errors - if they exist here then they existed at index time
-                    // and so on_script_error must have been set to `ignore`
+                    ignoredValues.addAll(values);
                 }
                 return formatter.apply(values);
             }

server/src/test/java/org/elasticsearch/index/get/DocumentFieldTests.java

@@ -36,9 +36,11 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXC
 public class DocumentFieldTests extends ESTestCase {
 
     public void testToXContent() {
-        DocumentField documentField = new DocumentField("field", Arrays.asList("value1", "value2"));
-        String output = Strings.toString(documentField);
+        DocumentField documentField = new DocumentField("field", Arrays.asList("value1", "value2"), Arrays.asList("ignored1", "ignored2"));
+        String output = Strings.toString(documentField.getValidValuesWriter());
         assertEquals("{\"field\":[\"value1\",\"value2\"]}", output);
+        String ignoredOutput = Strings.toString(documentField.getIgnoredValuesWriter());
+        assertEquals("{\"field\":[\"ignored1\",\"ignored2\"]}", ignoredOutput);
     }
 
     public void testEqualsAndHashcode() {
@@ -52,7 +54,12 @@ public class DocumentFieldTests extends ESTestCase {
         DocumentField documentField = tuple.v1();
         DocumentField expectedDocumentField = tuple.v2();
         boolean humanReadable = randomBoolean();
-        BytesReference originalBytes = toShuffledXContent(documentField, xContentType, ToXContent.EMPTY_PARAMS, humanReadable);
+        BytesReference originalBytes = toShuffledXContent(
+            documentField.getValidValuesWriter(),
+            xContentType,
+            ToXContent.EMPTY_PARAMS,
+            humanReadable
+        );
         //test that we can parse what we print out
         DocumentField parsedDocumentField;
         try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
@@ -65,7 +72,7 @@ public class DocumentFieldTests extends ESTestCase {
             assertNull(parser.nextToken());
         }
         assertEquals(expectedDocumentField, parsedDocumentField);
-        BytesReference finalBytes = toXContent(parsedDocumentField, xContentType, humanReadable);
+        BytesReference finalBytes = toXContent(parsedDocumentField.getValidValuesWriter(), xContentType, humanReadable);
         assertToXContentEquivalent(originalBytes, finalBytes, xContentType);
     }
 

server/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java

@@ -17,6 +17,7 @@ import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 
 import static org.elasticsearch.index.mapper.IdFieldMapper.ID_FIELD_DATA_DEPRECATION_MESSAGE;
@@ -85,7 +86,7 @@ public class IdFieldMapperTests extends MapperServiceTestCase {
             LeafReaderContext context = searcher.getIndexReader().leaves().get(0);
             lookup.source().setSegmentAndDocument(context, 0);
             valueFetcher.setNextReader(context);
-            assertEquals(List.of(id), valueFetcher.fetchValues(lookup.source()));
+            assertEquals(List.of(id), valueFetcher.fetchValues(lookup.source(), new ArrayList<>()));
         });
     }
 

server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java

@@ -178,9 +178,9 @@ public class KeyedFlattenedFieldTypeTests extends FieldTypeTestCase {
         SourceLookup lookup = new SourceLookup();
         lookup.setSource(Collections.singletonMap("field", sourceValue));
 
-        assertEquals(List.of("value"), fetcher.fetchValues(lookup));
+        assertEquals(List.of("value"), fetcher.fetchValues(lookup, new ArrayList<Object>()));
         lookup.setSource(Collections.singletonMap("field", null));
-        assertEquals(List.of(), fetcher.fetchValues(lookup));
+        assertEquals(List.of(), fetcher.fetchValues(lookup, new ArrayList<Object>()));
 
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> ft.valueFetcher(searchExecutionContext, "format"));
         assertEquals("Field [field.key] of type [flattened] doesn't support formats.", e.getMessage());

server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java

@@ -384,7 +384,8 @@ public class FieldFetcherTests extends MapperServiceTestCase {
             .array("field", "really_really_long_value")
             .endObject();
         fields = fetchFields(mapperService, source, "field");
-        assertFalse(fields.containsKey("field"));
+        assertThat(fields.get("field").getValues().size(), equalTo(0));
+        assertThat(fields.get("field").getIgnoredValues().size(), equalTo(1));
     }
 
     public void testFieldAliases() throws IOException {
@@ -862,14 +863,17 @@ public class FieldFetcherTests extends MapperServiceTestCase {
 
         // this should not return a field bc. f1 is malformed
         Map<String, DocumentField> fields = fetchFields(mapperService, source, List.of(new FieldAndFormat("*", null, true)));
-        assertThat(fields.size(), equalTo(0));
+        assertThat(fields.get("f1").getValues().size(), equalTo(0));
+        assertThat(fields.get("f1").getIgnoredValues().size(), equalTo(1));
 
         // and this should neither
         fields = fetchFields(mapperService, source, List.of(new FieldAndFormat("*", null, true)));
-        assertThat(fields.size(), equalTo(0));
+        assertThat(fields.get("f1").getValues().size(), equalTo(0));
+        assertThat(fields.get("f1").getIgnoredValues().size(), equalTo(1));
 
         fields = fetchFields(mapperService, source, List.of(new FieldAndFormat("f1", null, true)));
-        assertThat(fields.size(), equalTo(0));
+        assertThat(fields.get("f1").getValues().size(), equalTo(0));
+        assertThat(fields.get("f1").getIgnoredValues().size(), equalTo(1));
 
         // check this also does not overwrite with arrays
         source = XContentFactory.jsonBuilder().startObject()
@@ -877,7 +881,8 @@ public class FieldFetcherTests extends MapperServiceTestCase {
             .endObject();
 
         fields = fetchFields(mapperService, source, List.of(new FieldAndFormat("f1", null, true)));
-        assertThat(fields.size(), equalTo(0));
+        assertThat(fields.get("f1").getValues().size(), equalTo(0));
+        assertThat(fields.get("f1").getIgnoredValues().size(), equalTo(1));
     }
 
     public void testUnmappedFieldsWildcard() throws IOException {

test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java

@@ -13,6 +13,7 @@ import org.elasticsearch.search.lookup.SourceLookup;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Set;
@@ -53,7 +54,7 @@ public abstract class FieldTypeTestCase extends ESTestCase {
         ValueFetcher fetcher = fieldType.valueFetcher(searchExecutionContext, format);
         SourceLookup lookup = new SourceLookup();
         lookup.setSource(Collections.singletonMap(field, sourceValue));
-        return fetcher.fetchValues(lookup);
+        return fetcher.fetchValues(lookup, new ArrayList<>());
     }
 
     public static List<?> fetchSourceValues(MappedFieldType fieldType, Object... values) throws IOException {
@@ -64,6 +65,6 @@ public abstract class FieldTypeTestCase extends ESTestCase {
         ValueFetcher fetcher = fieldType.valueFetcher(searchExecutionContext, null);
         SourceLookup lookup = new SourceLookup();
         lookup.setSource(Collections.singletonMap(field, List.of(values)));
-        return fetcher.fetchValues(lookup);
+        return fetcher.fetchValues(lookup, new ArrayList<>());
     }
 }

test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java

@@ -335,7 +335,7 @@ public abstract class MapperTestCase extends MapperServiceTestCase {
             LeafReaderContext context = searcher.getIndexReader().leaves().get(0);
             lookup.source().setSegmentAndDocument(context, 0);
             valueFetcher.setNextReader(context);
-            result.set(valueFetcher.fetchValues(lookup.source()));
+            result.set(valueFetcher.fetchValues(lookup.source(), new ArrayList<>()));
         });
         return result.get();
     }
@@ -605,8 +605,8 @@ public abstract class MapperTestCase extends MapperServiceTestCase {
             sourceLookup.setSegmentAndDocument(ir.leaves().get(0), 0);
             docValueFetcher.setNextReader(ir.leaves().get(0));
             nativeFetcher.setNextReader(ir.leaves().get(0));
-            List<Object> fromDocValues = docValueFetcher.fetchValues(sourceLookup);
-            List<Object> fromNative = nativeFetcher.fetchValues(sourceLookup);
+            List<Object> fromDocValues = docValueFetcher.fetchValues(sourceLookup, new ArrayList<>());
+            List<Object> fromNative = nativeFetcher.fetchValues(sourceLookup, new ArrayList<>());
             /*
              * The native fetcher uses byte, short, etc but doc values always
              * uses long or double. This difference is fine because on the outside

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldMapper.java

@@ -79,8 +79,8 @@ public class DataTierFieldMapper extends MetadataFieldMapper {
 
             String tierPreference = getTierPreference(context);
             return tierPreference == null
-                ? lookup -> List.of()
-                : lookup -> List.of(tierPreference);
+                ? (lookup, ignoredValues) -> List.of()
+                : (lookup, ignoredValues) -> List.of(tierPreference);
         }
 
         /**

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldTypeTests.java

@@ -23,7 +23,9 @@ import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.search.lookup.SourceLookup;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.List;
 import java.util.function.Predicate;
 
 import static java.util.Collections.emptyMap;
@@ -88,11 +90,12 @@ public class DataTierFieldTypeTests extends MapperServiceTestCase {
         MappedFieldType ft = DataTierFieldMapper.DataTierFieldType.INSTANCE;
         SourceLookup lookup = new SourceLookup();
 
+        List<Object> ignoredValues = new ArrayList<>();
         ValueFetcher valueFetcher = ft.valueFetcher(createContext(), null);
-        assertEquals(singletonList("data_warm"), valueFetcher.fetchValues(lookup));
+        assertEquals(singletonList("data_warm"), valueFetcher.fetchValues(lookup, ignoredValues));
 
         ValueFetcher emptyValueFetcher = ft.valueFetcher(createContextWithoutSetting(), null);
-        assertTrue(emptyValueFetcher.fetchValues(lookup).isEmpty());
+        assertTrue(emptyValueFetcher.fetchValues(lookup, ignoredValues).isEmpty());
     }
 
     private SearchExecutionContext createContext() {

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java

@@ -279,7 +279,7 @@ public class EqlSearchResponse extends ActionResponse implements ToXContentObjec
                 builder.startObject(Fields.FIELDS);
                 for (DocumentField field : fetchFields.values()) {
                     if (field.getValues().size() > 0) {
-                        field.toXContent(builder, params);
+                        field.getValidValuesWriter().toXContent(builder, params);
                     }
                 }
                 builder.endObject();

x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java

@@ -141,8 +141,8 @@ public class ConstantKeywordFieldMapper extends FieldMapper {
             }
 
             return value == null
-                ? lookup -> List.of()
-                : lookup -> List.of(value);
+                ? (lookup, ignoredValues) -> List.of()
+                : (lookup, ignoredValues) -> List.of(value);
         }
 
 

x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java

@@ -17,6 +17,7 @@ import org.elasticsearch.index.mapper.ValueFetcher;
 import org.elasticsearch.search.lookup.SourceLookup;
 import org.elasticsearch.xpack.constantkeyword.mapper.ConstantKeywordFieldMapper.ConstantKeywordFieldType;
 
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
@@ -114,13 +115,14 @@ public class ConstantKeywordFieldTypeTests extends FieldTypeTestCase {
         SourceLookup nullValueLookup = new SourceLookup();
         nullValueLookup.setSource(Collections.singletonMap("field", null));
 
-        assertTrue(fetcher.fetchValues(missingValueLookup).isEmpty());
-        assertTrue(fetcher.fetchValues(nullValueLookup).isEmpty());
+        List<Object> ignoredValues = new ArrayList<>();
+        assertTrue(fetcher.fetchValues(missingValueLookup, ignoredValues).isEmpty());
+        assertTrue(fetcher.fetchValues(nullValueLookup, ignoredValues).isEmpty());
 
         MappedFieldType valued = new ConstantKeywordFieldMapper.ConstantKeywordFieldType("field", "foo");
         fetcher = valued.valueFetcher(null, null);
 
-        assertEquals(List.of("foo"), fetcher.fetchValues(missingValueLookup));
-        assertEquals(List.of("foo"), fetcher.fetchValues(nullValueLookup));
+        assertEquals(List.of("foo"), fetcher.fetchValues(missingValueLookup, ignoredValues));
+        assertEquals(List.of("foo"), fetcher.fetchValues(nullValueLookup, ignoredValues));
     }
 }