Synthetic _source: support ignore_above (#89466)

This allows you to use `ignore_above` with `keyword` fields in synthetic
source. Ignored values are stored in a "backup" stored field and added
to the end of the list of results. This makes `ignore_above` work pretty
much the same way as it does when you don't have synthetic source. The
only difference is the order of the results. But synthetic source
changes the order of results anyway. That should be fine.

docs/changelog/89466.yaml

@@ -0,0 +1,5 @@
+pr: 89466
+summary: "Synthetic _source: support `ignore_above`"
+area: "TSDB"
+type: feature
+issues: []

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml

@@ -33,6 +33,7 @@ keyword:
   - match:
       _source:
         kwd: foo
+#  - is_false: fields  TODO fix me
 
 ---
 fetch without refresh also produces synthetic source:
@@ -63,6 +64,7 @@ fetch without refresh also produces synthetic source:
         refresh: false # no refreshing!
         body:
           obj.kwd: foo
+#  - is_false: fields  TODO fix me
 
   - do:
       get:
@@ -76,6 +78,7 @@ fetch without refresh also produces synthetic source:
       _source: # synthetic source will convert the dotted field names into an object, even when loading from the translog
         obj:
           kwd: foo
+#  - is_false: fields  TODO fix me
 
 ---
 force_synthetic_source_ok:
@@ -103,6 +106,7 @@ force_synthetic_source_ok:
         refresh: true
         body:
           obj.kwd: foo
+#  - is_false: fields  TODO fix me
 
   # When _source is used in the fetch the original _source is perfect
   - do:
@@ -123,6 +127,7 @@ force_synthetic_source_ok:
       _source:
         obj:
           kwd: foo
+#  - is_false: fields  TODO fix me
 
 ---
 force_synthetic_source_bad_mapping:
@@ -185,6 +190,7 @@ stored text:
               text:
                 type: text
                 store: true
+#  - is_false: fields  TODO fix me
 
   - do:
       index:
@@ -193,6 +199,7 @@ stored text:
         refresh: true
         body:
           text: the quick brown fox
+#  - is_false: fields  TODO fix me
 
   - do:
       get:
@@ -205,6 +212,7 @@ stored text:
   - match:
       _source:
         text: the quick brown fox
+#  - is_false: fields  TODO fix me
 
 ---
 stored keyword:
@@ -223,6 +231,7 @@ stored keyword:
               kwd:
                 type: keyword
                 store: true
+#  - is_false: fields  TODO fix me
 
   - do:
       index:
@@ -231,6 +240,7 @@ stored keyword:
         refresh: true
         body:
           kwd: the quick brown fox
+#  - is_false: fields  TODO fix me
 
   - do:
       get:
@@ -243,3 +253,172 @@ stored keyword:
   - match:
       _source:
         kwd: the quick brown fox
+#  - is_false: fields  TODO fix me
+
+---
+doc values keyword with ignore_above:
+  - skip:
+      version: " - 8.4.99"
+      reason: introduced in 8.5.0
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              kwd:
+                type: keyword
+                ignore_above: 10
+
+  - do:
+      index:
+        index:   test
+        id:      1
+        refresh: true
+        body:
+          kwd: the quick brown fox
+  - do:
+      index:
+        index:   test
+        id:      2
+        refresh: true
+        body:
+          kwd: short
+  - do:
+      index:
+        index:   test
+        id:      3
+        refresh: true
+        body:
+          kwd:
+            - jumped over the lazy dog
+            - short
+
+  - do:
+      get:
+        index: test
+        id:    1
+  - match: {_index: "test"}
+  - match: {_id: "1"}
+  - match: {_version: 1}
+  - match: {found: true}
+  - match:
+      _source:
+        kwd: the quick brown fox
+#  - is_false: fields  TODO fix me
+
+  - do:
+      get:
+        index: test
+        id:    2
+  - match: {_index: "test"}
+  - match: {_id: "2"}
+  - match: {_version: 1}
+  - match: {found: true}
+  - match:
+      _source:
+        kwd: short
+#  - is_false: fields  TODO fix me
+
+  - do:
+      get:
+        index: test
+        id:    3
+  - match: {_index: "test"}
+  - match: {_id: "3"}
+  - match: {_version: 1}
+  - match: {found: true}
+  - match:
+      _source:
+        kwd:
+          - short
+          - jumped over the lazy dog # fields saved by ignore_above are returned after doc values fields
+#  - is_false: fields  TODO fix me
+
+---
+stored keyword with ignore_above:
+  - skip:
+      version: " - 8.4.99"
+      reason: introduced in 8.5.0
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              kwd:
+                type: keyword
+                doc_values: false
+                store: true
+                ignore_above: 10
+
+  - do:
+      index:
+        index:   test
+        id:      1
+        refresh: true
+        body:
+          kwd: the quick brown fox
+  - do:
+      index:
+        index:   test
+        id:      2
+        refresh: true
+        body:
+          kwd: short
+  - do:
+      index:
+        index:   test
+        id:      3
+        refresh: true
+        body:
+          kwd:
+            - jumped over the lazy dog
+            - short
+
+  - do:
+      get:
+        index: test
+        id:    1
+  - match: {_index: "test"}
+  - match: {_id: "1"}
+  - match: {_version: 1}
+  - match: {found: true}
+  - match:
+      _source:
+        kwd: the quick brown fox
+#  - is_false: fields  TODO fix me
+
+  - do:
+      get:
+        index: test
+        id:    2
+  - match: {_index: "test"}
+  - match: {_id: "2"}
+  - match: {_version: 1}
+  - match: {found: true}
+  - match:
+      _source:
+        kwd: short
+#  - is_false: fields  TODO fix me
+
+  - do:
+      get:
+        index: test
+        id:    3
+  - match: {_index: "test"}
+  - match: {_id: "3"}
+  - match: {_version: 1}
+  - match: {found: true}
+  - match:
+      _source:
+        kwd:
+          - short
+          - jumped over the lazy dog # fields saved by ignore_above are returned after doc values fields
+#  - is_false: fields  TODO fix me

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mget/90_synthetic_source.yml

@@ -212,5 +212,5 @@ force_synthetic_source_bad_mapping:
         force_synthetic_source: true
         body:
           ids: [ 1, 2 ]
-  - match: {docs.0.error.reason: "field [text] of type [text] doesn't support synthetic source unless it is stored or has a sub-field of type [keyword] with doc values or stored and without ignore_above or a normalizer"}
-  - match: {docs.1.error.reason: "field [text] of type [text] doesn't support synthetic source unless it is stored or has a sub-field of type [keyword] with doc values or stored and without ignore_above or a normalizer"}
+  - match: {docs.0.error.reason: "field [text] of type [text] doesn't support synthetic source unless it is stored or has a sub-field of type [keyword] with doc values or stored and without a normalizer"}
+  - match: {docs.1.error.reason: "field [text] of type [text] doesn't support synthetic source unless it is stored or has a sub-field of type [keyword] with doc values or stored and without a normalizer"}

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/400_synthetic_source.yml

@@ -29,6 +29,7 @@ keyword:
           query:
             ids:
               values: [1]
+  - is_false: hits.hits.0.fields
   - match:
       hits.hits.0._source:
         kwd: foo
@@ -66,6 +67,7 @@ stored text:
           query:
             ids:
               values: [1]
+  - is_false: hits.hits.0.fields
   - match:
       hits.hits.0._source:
         text: the quick brown fox
@@ -103,6 +105,7 @@ stored keyword:
           query:
             ids:
               values: [1]
+  - is_false: hits.hits.0.fields
   - match:
       hits.hits.0._source:
         kwd: the quick brown fox
@@ -140,6 +143,8 @@ stored keyword without sibling fields:
         index: test
         body:
           sort: s
+  - is_false: hits.hits.0.fields
+  - is_false: hits.hits.1.fields
   - match:
       hits.hits.0._source:
         kwd: the quick brown fox
@@ -252,3 +257,141 @@ force_synthetic_source_bad_mapping:
           query:
             ids:
               values: [1]
+
+---
+doc values keyword with ignore_above:
+  - skip:
+      version: " - 8.4.99"
+      reason: introduced in 8.5.0
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              kwd:
+                type: keyword
+                ignore_above: 10
+
+  - do:
+      index:
+        index:   test
+        id:      1
+        refresh: true
+        body:
+          s: 1
+          kwd: the quick brown fox
+  - do:
+      index:
+        index:   test
+        id:      2
+        refresh: true
+        body:
+          s: 2
+          kwd: short
+  - do:
+      index:
+        index:   test
+        id:      3
+        refresh: true
+        body:
+          s: 3
+          kwd:
+            - jumped over the lazy dog
+            - short
+
+  - do:
+      search:
+        index: test
+        body:
+          sort: s
+  - is_false: hits.hits.0.fields
+  - is_false: hits.hits.1.fields
+  - is_false: hits.hits.2.fields
+  - match:
+      hits.hits.0._source:
+        s: 1
+        kwd: the quick brown fox
+  - match:
+      hits.hits.1._source:
+        s: 2
+        kwd: short
+  - match:
+      hits.hits.2._source:
+        s: 3
+        kwd:
+          - short
+          - jumped over the lazy dog # fields saved by ignore_above are returned after doc values fields
+
+---
+stored keyword with ignore_above:
+  - skip:
+      version: " - 8.4.99"
+      reason: introduced in 8.5.0
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              kwd:
+                type: keyword
+                doc_values: false
+                store: true
+                ignore_above: 10
+
+  - do:
+      index:
+        index:   test
+        id:      1
+        refresh: true
+        body:
+          s: 1
+          kwd: the quick brown fox
+  - do:
+      index:
+        index:   test
+        id:      2
+        refresh: true
+        body:
+          s: 2
+          kwd: short
+  - do:
+      index:
+        index:   test
+        id:      3
+        refresh: true
+        body:
+          s: 3
+          kwd:
+            - jumped over the lazy dog
+            - short
+
+  - do:
+      search:
+        index: test
+        body:
+          sort: s
+  - is_false: hits.hits.0.fields
+  - is_false: hits.hits.1.fields
+  - is_false: hits.hits.2.fields
+  - match:
+      hits.hits.0._source:
+        s: 1
+        kwd: the quick brown fox
+  - match:
+      hits.hits.1._source:
+        s: 2
+        kwd: short
+  - match:
+      hits.hits.2._source:
+        s: 3
+        kwd:
+          - short
+          - jumped over the lazy dog # fields saved by ignore_above are returned after doc values fields

server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java

@@ -436,6 +436,14 @@ public abstract class DocumentParserContext {
         return null;
     }
 
+    /**
+     * Is this index configured to use synthetic source?
+     */
+    public final boolean isSyntheticSource() {
+        SourceFieldMapper sft = mappingLookup.getMapping().getMetadataMapperByClass(SourceFieldMapper.class);
+        return sft == null ? false : sft.isSynthetic();
+    }
+
     // XContentParser that wraps an existing parser positioned on a value,
     // and a field name, and returns a stream that looks like { 'field' : 'value' }
     private static class CopyToParser extends FilterXContentParserWrapper {

server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java

@@ -553,7 +553,7 @@ public class IpFieldMapper extends FieldMapper {
                 "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to"
             );
         }
-        return new SortedSetDocValuesSyntheticFieldLoader(name(), simpleName()) {
+        return new SortedSetDocValuesSyntheticFieldLoader(name(), simpleName(), null) {
             @Override
             protected BytesRef convert(BytesRef value) {
                 byte[] bytes = Arrays.copyOfRange(value.bytes, value.offset, value.offset + value.length);

server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java

@@ -15,6 +15,7 @@ import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.document.StoredField;
 import org.apache.lucene.index.FilteredTermsEnum;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexReader;
@@ -62,6 +63,7 @@ import org.elasticsearch.search.runtime.StringScriptFieldPrefixQuery;
 import org.elasticsearch.search.runtime.StringScriptFieldRegexpQuery;
 import org.elasticsearch.search.runtime.StringScriptFieldTermQuery;
 import org.elasticsearch.search.runtime.StringScriptFieldWildcardQuery;
+import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
@@ -945,6 +947,10 @@ public final class KeywordFieldMapper extends FieldMapper {
 
         if (value.length() > fieldType().ignoreAbove()) {
             context.addIgnoredField(name());
+            if (context.isSyntheticSource()) {
+                // Save a copy of the field so synthetic source can load it
+                context.doc().add(new StoredField(originalName(), new BytesRef(value)));
+            }
             return;
         }
 
@@ -1046,6 +1052,15 @@ public final class KeywordFieldMapper extends FieldMapper {
         return normalizerName != null;
     }
 
+    /**
+     * The name used to store "original" that have been ignored
+     * by {@link KeywordFieldType#ignoreAbove()} so that they can be rebuilt
+     * for synthetic source.
+     */
+    private String originalName() {
+        return name() + "._original";
+    }
+
     @Override
     public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
         return syntheticFieldLoader(simpleName());
@@ -1055,11 +1070,6 @@ public final class KeywordFieldMapper extends FieldMapper {
         if (hasScript()) {
             return SourceLoader.SyntheticFieldLoader.NOTHING;
         }
-        if (fieldType().ignoreAbove() != Defaults.IGNORE_ABOVE) {
-            throw new IllegalArgumentException(
-                "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares ignore_above"
-            );
-        }
         if (copyTo.copyToFields().isEmpty() != true) {
             throw new IllegalArgumentException(
                 "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to"
@@ -1071,10 +1081,15 @@ public final class KeywordFieldMapper extends FieldMapper {
             );
         }
         if (fieldType.stored()) {
-            return new StringStoredFieldFieldLoader(name(), simpleName) {
+            return new StringStoredFieldFieldLoader(
+                name(),
+                simpleName,
+                fieldType().ignoreAbove == Defaults.IGNORE_ABOVE ? null : originalName()
+            ) {
                 @Override
-                public void load(List<Object> values) {
-                    super.load(values.stream().map(fieldType()::valueForDisplay).toList());
+                protected void write(XContentBuilder b, Object value) throws IOException {
+                    BytesRef ref = (BytesRef) value;
+                    b.utf8Value(ref.bytes, ref.offset, ref.length);
                 }
             };
         }
@@ -1087,7 +1102,11 @@ public final class KeywordFieldMapper extends FieldMapper {
                     + "] doesn't support synthetic source because it doesn't have doc values and isn't stored"
             );
         }
-        return new SortedSetDocValuesSyntheticFieldLoader(name(), simpleName) {
+        return new SortedSetDocValuesSyntheticFieldLoader(
+            name(),
+            simpleName,
+            fieldType().ignoreAbove == Defaults.IGNORE_ABOVE ? null : originalName()
+        ) {
             @Override
             protected BytesRef convert(BytesRef value) {
                 return value;

server/src/main/java/org/elasticsearch/index/mapper/SortedSetDocValuesSyntheticFieldLoader.java

@@ -13,15 +13,19 @@ import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.SortedDocValues;
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.List;
 import java.util.Map;
 import java.util.stream.Stream;
 
+import static java.util.Collections.emptyList;
+
 /**
  * Load {@code _source} fields from {@link SortedSetDocValues}.
  */
@@ -30,23 +34,37 @@ public abstract class SortedSetDocValuesSyntheticFieldLoader implements SourceLo
 
     private final String name;
     private final String simpleName;
-    private Values values = NO_VALUES;
+    private DocValuesFieldValues docValues = NO_VALUES;
 
-    public SortedSetDocValuesSyntheticFieldLoader(String name, String simpleName) {
+    /**
+     * Optionally loads stored fields values.
+     */
+    @Nullable
+    private final String storedValuesName;
+    private List<Object> storedValues = emptyList();
+
+    /**
+     * Build a loader from doc values and, optionally, a stored field.
+     * @param name the name of the field to load from doc values
+     * @param simpleName the name to give the field in the rendered {@code _source}
+     * @param storedValuesName the name of a stored field to load or null if there aren't any stored field for this field
+     */
+    public SortedSetDocValuesSyntheticFieldLoader(String name, String simpleName, @Nullable String storedValuesName) {
         this.name = name;
         this.simpleName = simpleName;
+        this.storedValuesName = storedValuesName;
     }
 
     @Override
     public Stream<Map.Entry<String, StoredFieldLoader>> storedFieldLoaders() {
-        return Stream.of();
+        return storedValuesName == null ? Stream.of() : Stream.of(Map.entry(storedValuesName, values -> this.storedValues = values));
     }
 
     @Override
     public DocValuesLoader docValuesLoader(LeafReader reader, int[] docIdsInLeaf) throws IOException {
         SortedSetDocValues dv = DocValues.getSortedSet(reader, name);
         if (dv.getValueCount() == 0) {
-            values = NO_VALUES;
+            docValues = NO_VALUES;
             return null;
         }
         if (docIdsInLeaf.length > 1) {
@@ -58,58 +76,74 @@ public abstract class SortedSetDocValuesSyntheticFieldLoader implements SourceLo
             SortedDocValues singleton = DocValues.unwrapSingleton(dv);
             if (singleton != null) {
                 SingletonDocValuesLoader loader = buildSingletonDocValuesLoader(singleton, docIdsInLeaf);
-                values = loader == null ? NO_VALUES : loader;
+                docValues = loader == null ? NO_VALUES : loader;
                 return loader;
             }
         }
         ImmediateDocValuesLoader loader = new ImmediateDocValuesLoader(dv);
-        values = loader;
+        docValues = loader;
         return loader;
     }
 
     @Override
     public boolean hasValue() {
-        return values.count() > 0;
+        return docValues.count() > 0 || storedValues.isEmpty() == false;
     }
 
     @Override
     public void write(XContentBuilder b) throws IOException {
-        switch (values.count()) {
+        int total = docValues.count() + storedValues.size();
+        switch (total) {
             case 0:
                 return;
             case 1:
                 b.field(simpleName);
-                values.write(b);
+                if (docValues.count() > 0) {
+                    assert docValues.count() == 1;
+                    assert storedValues.isEmpty();
+                    docValues.write(b);
+                } else {
+                    assert docValues.count() == 0;
+                    assert storedValues.size() == 1;
+                    BytesRef ref = (BytesRef) storedValues.get(0);
+                    b.utf8Value(ref.bytes, ref.offset, ref.length);
+                    storedValues = emptyList();
+                }
                 return;
             default:
                 b.startArray(simpleName);
-                values.write(b);
+                docValues.write(b);
+                for (Object v : storedValues) {
+                    BytesRef ref = (BytesRef) v;
+                    b.utf8Value(ref.bytes, ref.offset, ref.length);
+                }
+                storedValues = emptyList();
                 b.endArray();
                 return;
         }
     }
 
-    private interface Values {
+    private interface DocValuesFieldValues {
         int count();
 
         void write(XContentBuilder b) throws IOException;
     }
 
-    private static final Values NO_VALUES = new Values() {
+    private static final DocValuesFieldValues NO_VALUES = new DocValuesFieldValues() {
         @Override
         public int count() {
             return 0;
         }
 
         @Override
-        public void write(XContentBuilder b) throws IOException {}
+        public void write(XContentBuilder b) {}
     };
 
     /**
      * Load ordinals in line with populating the doc and immediately
      * convert from ordinals into {@link BytesRef}s.
      */
-    private class ImmediateDocValuesLoader implements DocValuesLoader, Values {
+    private class ImmediateDocValuesLoader implements DocValuesLoader, DocValuesFieldValues {
         private final SortedSetDocValues dv;
         private boolean hasValue;
 
@@ -129,7 +163,9 @@ public abstract class SortedSetDocValuesSyntheticFieldLoader implements SourceLo
 
         @Override
         public void write(XContentBuilder b) throws IOException {
-            assert hasValue;
+            if (hasValue == false) {
+                return;
+            }
             for (int i = 0; i < dv.docValueCount(); i++) {
                 BytesRef c = convert(dv.lookupOrd(dv.nextOrd()));
                 b.utf8Value(c.bytes, c.offset, c.length);
@@ -183,7 +219,7 @@ public abstract class SortedSetDocValuesSyntheticFieldLoader implements SourceLo
         return new SingletonDocValuesLoader(docIdsInLeaf, ords, uniqueOrds, converted);
     }
 
-    private class SingletonDocValuesLoader implements DocValuesLoader, Values {
+    private static class SingletonDocValuesLoader implements DocValuesLoader, DocValuesFieldValues {
         private final int[] docIdsInLeaf;
         private final int[] ords;
         private final int[] uniqueOrds;
@@ -216,7 +252,9 @@ public abstract class SortedSetDocValuesSyntheticFieldLoader implements SourceLo
 
         @Override
         public void write(XContentBuilder b) throws IOException {
-            assert ords[idx] >= 0;
+            if (ords[idx] < 0) {
+                return;
+            }
             int convertedIdx = Arrays.binarySearch(uniqueOrds, ords[idx]);
             if (convertedIdx < 0) {
                 throw new IllegalStateException("received unexpected ord [" + ords[idx] + "]. Expected " + Arrays.toString(uniqueOrds));

server/src/main/java/org/elasticsearch/index/mapper/StringStoredFieldFieldLoader.java

@@ -9,6 +9,7 @@
 package org.elasticsearch.index.mapper;
 
 import org.apache.lucene.index.LeafReader;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
@@ -16,52 +17,74 @@ import java.util.List;
 import java.util.Map;
 import java.util.stream.Stream;
 
-public class StringStoredFieldFieldLoader
-    implements
-        SourceLoader.SyntheticFieldLoader,
-        SourceLoader.SyntheticFieldLoader.StoredFieldLoader {
+import static java.util.Collections.emptyList;
+
+public abstract class StringStoredFieldFieldLoader implements SourceLoader.SyntheticFieldLoader {
     private final String name;
     private final String simpleName;
-    private List<Object> values;
+    private List<Object> values = emptyList();
+
+    @Nullable
+    private final String extraStoredName;
+    private List<Object> extraValues = emptyList();
 
-    public StringStoredFieldFieldLoader(String name, String simpleName) {
+    public StringStoredFieldFieldLoader(String name, String simpleName, @Nullable String extraStoredName) {
         this.name = name;
         this.simpleName = simpleName;
+        this.extraStoredName = extraStoredName;
     }
 
     @Override
-    public Stream<Map.Entry<String, StoredFieldLoader>> storedFieldLoaders() {
-        return Stream.of(Map.entry(name, this));
-    }
-
-    @Override
-    public void load(List<Object> values) {
-        this.values = values;
+    public final Stream<Map.Entry<String, StoredFieldLoader>> storedFieldLoaders() {
+        Stream<Map.Entry<String, StoredFieldLoader>> standard = Stream.of(Map.entry(name, values -> this.values = values));
+        if (extraStoredName == null) {
+            return standard;
+        }
+        return Stream.concat(standard, Stream.of(Map.entry(extraStoredName, values -> this.extraValues = values)));
     }
 
     @Override
-    public boolean hasValue() {
-        return values != null && values.isEmpty() == false;
+    public final boolean hasValue() {
+        return values.isEmpty() == false || extraValues.isEmpty() == false;
     }
 
     @Override
-    public void write(XContentBuilder b) throws IOException {
-        if (values == null || values.isEmpty()) {
-            return;
-        }
-        if (values.size() == 1) {
-            b.field(simpleName, values.get(0).toString());
-            values = null;
-            return;
+    public final void write(XContentBuilder b) throws IOException {
+        int size = values.size() + extraValues.size();
+        switch (size) {
+            case 0:
+                return;
+            case 1:
+                b.field(simpleName);
+                if (values.size() > 0) {
+                    assert values.size() == 1;
+                    assert extraValues.isEmpty();
+                    write(b, values.get(0));
+                } else {
+                    assert values.isEmpty();
+                    assert extraValues.size() == 1;
+                    write(b, extraValues.get(0));
+                }
+                values = emptyList();
+                extraValues = emptyList();
+                return;
+            default:
+                b.startArray(simpleName);
+                for (Object value : values) {
+                    write(b, value);
+                }
+                for (Object value : extraValues) {
+                    write(b, value);
+                }
+                b.endArray();
+                values = emptyList();
+                extraValues = emptyList();
+                return;
         }
-        b.startArray(simpleName);
-        for (Object value : values) {
-            b.value(value.toString());
-        }
-        b.endArray();
-        values = null;
     }
 
+    protected abstract void write(XContentBuilder b, Object value) throws IOException;
+
     @Override
     public final DocValuesLoader docValuesLoader(LeafReader reader, int[] docIdsInLeaf) throws IOException {
         return null;

server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java

@@ -1288,14 +1288,17 @@ public class TextFieldMapper extends FieldMapper {
             );
         }
         if (store) {
-            return new StringStoredFieldFieldLoader(name(), simpleName());
+            return new StringStoredFieldFieldLoader(name(), simpleName(), null) {
+                @Override
+                protected void write(XContentBuilder b, Object value) throws IOException {
+                    b.value((String) value);
+                }
+            };
         }
         for (Mapper sub : this) {
             if (sub.typeName().equals(KeywordFieldMapper.CONTENT_TYPE)) {
                 KeywordFieldMapper kwd = (KeywordFieldMapper) sub;
-                if (kwd.hasNormalizer() == false
-                    && kwd.fieldType().ignoreAbove() == KeywordFieldMapper.Defaults.IGNORE_ABOVE
-                    && (kwd.fieldType().hasDocValues() || kwd.fieldType().isStored())) {
+                if (kwd.hasNormalizer() == false && (kwd.fieldType().hasDocValues() || kwd.fieldType().isStored())) {
 
                     return kwd.syntheticFieldLoader(simpleName());
                 }
@@ -1305,7 +1308,7 @@ public class TextFieldMapper extends FieldMapper {
             String.format(
                 Locale.ROOT,
                 "field [%s] of type [%s] doesn't support synthetic source unless it is stored or has a sub-field of"
-                    + " type [keyword] with doc values or stored and without ignore_above or a normalizer",
+                    + " type [keyword] with doc values or stored and without a normalizer",
                 name(),
                 typeName()
             )

server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java

@@ -242,6 +242,12 @@ public class FetchPhase {
                 context.fetchSourceContext(FetchSourceContext.FETCH_SOURCE);
             }
             boolean loadSource = sourceRequired(context);
+            if (loadSource) {
+                if (false == sourceLoader.requiredStoredFields().isEmpty()) {
+                    // add the stored fields needed to load the source mapping to an empty set so they aren't returned
+                    sourceLoader.requiredStoredFields().forEach(fieldName -> storedToRequestedFields.putIfAbsent(fieldName, Set.of()));
+                }
+            }
             return StoredFieldLoader.create(loadSource, sourceLoader.requiredStoredFields());
         } else if (storedFieldsContext.fetchFields() == false) {
             // disable stored fields entirely
@@ -474,6 +480,7 @@ public class FetchPhase {
     public static List<Object> processStoredField(Function<String, MappedFieldType> fieldTypeLookup, String field, List<Object> input) {
         MappedFieldType ft = fieldTypeLookup.apply(field);
         if (ft == null) {
+            // TODO remove this once we've stopped calling it for accidentally loaded fields
             return input;
         }
         return input.stream().map(ft::valueForDisplay).collect(Collectors.toList());

server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java

@@ -44,8 +44,10 @@ import org.elasticsearch.script.StringFieldScript;
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -623,16 +625,20 @@ public class KeywordFieldMapperTests extends MapperTestCase {
 
     @Override
     protected SyntheticSourceSupport syntheticSourceSupport() {
-        return new KeywordSyntheticSourceSupport(randomBoolean(), usually() ? null : randomAlphaOfLength(2));
+        return new KeywordSyntheticSourceSupport(randomBoolean(), usually() ? null : randomAlphaOfLength(2), true);
     }
 
     static class KeywordSyntheticSourceSupport implements SyntheticSourceSupport {
+        private final Integer ignoreAbove = randomBoolean() ? null : between(10, 100);
+        private final boolean allIgnored = ignoreAbove != null && rarely();
         private final boolean store;
         private final String nullValue;
+        private final boolean exampleSortsUsingIgnoreAbove;
 
-        KeywordSyntheticSourceSupport(boolean store, String nullValue) {
+        KeywordSyntheticSourceSupport(boolean store, String nullValue, boolean exampleSortsUsingIgnoreAbove) {
             this.store = store;
             this.nullValue = nullValue;
+            this.exampleSortsUsingIgnoreAbove = exampleSortsUsingIgnoreAbove;
         }
 
         @Override
@@ -643,9 +649,17 @@ public class KeywordFieldMapperTests extends MapperTestCase {
             }
             List<Tuple<String, String>> values = randomList(1, maxValues, this::generateValue);
             List<String> in = values.stream().map(Tuple::v1).toList();
-            List<String> outList = store
-                ? values.stream().map(Tuple::v2).toList()
-                : values.stream().map(Tuple::v2).collect(Collectors.toSet()).stream().sorted().toList();
+            List<String> outPrimary = new ArrayList<>();
+            List<String> outExtraValues = new ArrayList<>();
+            values.stream().map(Tuple::v2).forEach(v -> {
+                if (exampleSortsUsingIgnoreAbove && ignoreAbove != null && v.length() > ignoreAbove) {
+                    outExtraValues.add(v);
+                } else {
+                    outPrimary.add(v);
+                }
+            });
+            List<String> outList = store ? outPrimary : new HashSet<>(outPrimary).stream().sorted().collect(Collectors.toList());
+            outList.addAll(outExtraValues);
             Object out = outList.size() == 1 ? outList.get(0) : outList;
             return new SyntheticSourceExample(in, out, this::mapping);
         }
@@ -654,7 +668,11 @@ public class KeywordFieldMapperTests extends MapperTestCase {
             if (nullValue != null && randomBoolean()) {
                 return Tuple.tuple(null, nullValue);
             }
-            String v = randomAlphaOfLength(5);
+            int length = 5;
+            if (ignoreAbove != null && (allIgnored || randomBoolean())) {
+                length = ignoreAbove + 5;
+            }
+            String v = randomAlphaOfLength(length);
             return Tuple.tuple(v, v);
         }
 
@@ -663,6 +681,9 @@ public class KeywordFieldMapperTests extends MapperTestCase {
             if (nullValue != null) {
                 b.field("null_value", nullValue);
             }
+            if (ignoreAbove != null) {
+                b.field("ignore_above", ignoreAbove);
+            }
             if (store) {
                 b.field("store", true);
                 if (randomBoolean()) {
@@ -681,10 +702,6 @@ public class KeywordFieldMapperTests extends MapperTestCase {
                     ),
                     b -> b.field("type", "keyword").field("doc_values", false)
                 ),
-                new SyntheticSourceInvalidExample(
-                    equalTo("field [field] of type [keyword] doesn't support synthetic source because it declares ignore_above"),
-                    b -> b.field("type", "keyword").field("ignore_above", 10)
-                ),
                 new SyntheticSourceInvalidExample(
                     equalTo("field [field] of type [keyword] doesn't support synthetic source because it declares a normalizer"),
                     b -> b.field("type", "keyword").field("normalizer", "lowercase")

server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java

@@ -42,7 +42,7 @@ public class SourceLoaderTests extends MapperServiceTestCase {
             e.getMessage(),
             equalTo(
                 "field [txt] of type [text] doesn't support synthetic source unless it is stored or has a sub-field "
-                    + "of type [keyword] with doc values or stored and without ignore_above or a normalizer"
+                    + "of type [keyword] with doc values or stored and without a normalizer"
             )
         );
     }

server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java

@@ -1098,11 +1098,15 @@ public class TextFieldMapperTests extends MapperTestCase {
         boolean storeTextField = randomBoolean();
         boolean storedKeywordField = storeTextField || randomBoolean();
         String nullValue = storeTextField || usually() ? null : randomAlphaOfLength(2);
+        KeywordFieldMapperTests.KeywordSyntheticSourceSupport keywordSupport = new KeywordFieldMapperTests.KeywordSyntheticSourceSupport(
+            storedKeywordField,
+            nullValue,
+            false == storeTextField
+        );
         return new SyntheticSourceSupport() {
             @Override
             public SyntheticSourceExample example(int maxValues) {
-                SyntheticSourceExample delegate = new KeywordFieldMapperTests.KeywordSyntheticSourceSupport(storedKeywordField, nullValue)
-                    .example(maxValues);
+                SyntheticSourceExample delegate = keywordSupport.example(maxValues);
                 if (storeTextField) {
                     return new SyntheticSourceExample(
                         delegate.inputValue(),
@@ -1126,7 +1130,7 @@ public class TextFieldMapperTests extends MapperTestCase {
             public List<SyntheticSourceInvalidExample> invalidExample() throws IOException {
                 Matcher<String> err = equalTo(
                     "field [field] of type [text] doesn't support synthetic source unless it is stored or"
-                        + " has a sub-field of type [keyword] with doc values or stored and without ignore_above or a normalizer"
+                        + " has a sub-field of type [keyword] with doc values or stored and without a normalizer"
                 );
                 return List.of(
                     new SyntheticSourceInvalidExample(err, TextFieldMapperTests.this::minimalMapping),
@@ -1140,17 +1144,6 @@ public class TextFieldMapperTests extends MapperTestCase {
                         }
                         b.endObject();
                     }),
-                    new SyntheticSourceInvalidExample(err, b -> {
-                        b.field("type", "text");
-                        b.startObject("fields");
-                        {
-                            b.startObject("kwd");
-                            b.field("type", "keyword");
-                            b.field("ignore_above", 10);
-                            b.endObject();
-                        }
-                        b.endObject();
-                    }),
                     new SyntheticSourceInvalidExample(err, b -> {
                         b.field("type", "text");
                         b.startObject("fields");

x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java

@@ -406,7 +406,7 @@ public class VersionStringFieldMapper extends FieldMapper {
                 "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to"
             );
         }
-        return new SortedSetDocValuesSyntheticFieldLoader(name(), simpleName()) {
+        return new SortedSetDocValuesSyntheticFieldLoader(name(), simpleName(), null) {
             @Override
             protected BytesRef convert(BytesRef value) {
                 return VersionEncoder.decodeVersion(value);