Natively store synthetic source array offsets for numeric fields (#124594)

This patch builds on the work in #122999 and #113757 to natively store
array offsets for numeric fields instead of falling back to ignored source
when `source_keep_mode: arrays`.

docs/changelog/124594.yaml

@@ -0,0 +1,5 @@
+pr: 124594
+summary: Store arrays offsets for numeric fields natively with synthetic source
+area: Mapping
+type: enhancement
+issues: []

server/src/main/java/org/elasticsearch/index/IndexVersions.java

@@ -152,6 +152,7 @@ public class IndexVersions {
     public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_KEYWORD = def(9_013_0_00, Version.LUCENE_10_1_0);
     public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_IP = def(9_014_0_00, Version.LUCENE_10_1_0);
     public static final IndexVersion ADD_RESCORE_PARAMS_TO_QUANTIZED_VECTORS = def(9_015_0_00, Version.LUCENE_10_1_0);
+    public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_NUMBER = def(9_016_0_00, Version.LUCENE_10_1_0);
     /*
      * STOP! READ THIS FIRST! No, really,
      *        ____ _____ ___  ____  _        ____  _____    _    ____    _____ _   _ ___ ____    _____ ___ ____  ____ _____ _

server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java

@@ -353,7 +353,8 @@ final class DynamicFieldsBuilder {
                     ScriptCompiler.NONE,
                     context.indexSettings().getSettings(),
                     context.indexSettings().getIndexVersionCreated(),
-                    context.indexSettings().getMode()
+                    context.indexSettings().getMode(),
+                    context.indexSettings().sourceKeepMode()
                 ),
                 context
             );
@@ -371,7 +372,8 @@ final class DynamicFieldsBuilder {
                     ScriptCompiler.NONE,
                     context.indexSettings().getSettings(),
                     context.indexSettings().getIndexVersionCreated(),
-                    context.indexSettings().getMode()
+                    context.indexSettings().getMode(),
+                    context.indexSettings().sourceKeepMode()
                 ),
                 context
             );

server/src/main/java/org/elasticsearch/index/mapper/FieldArrayContext.java

@@ -97,6 +97,7 @@ public class FieldArrayContext {
             && sourceKeepMode == Mapper.SourceKeepMode.ARRAYS
             && hasDocValues
             && isStored == false
+            && context.isInNestedContext() == false
             && fieldMapperBuilder.copyTo.copyToFields().isEmpty()
             && fieldMapperBuilder.multiFieldsBuilder.hasMultiFields() == false
             && indexVersionSupportStoringArraysNatively(indexCreatedVersion, minSupportedVersionMain)) {

server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java

@@ -37,6 +37,7 @@ import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.fielddata.FieldDataContext;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
@@ -70,6 +71,7 @@ import org.elasticsearch.xcontent.XContentParser.Token;
 import java.io.IOException;
 import java.math.BigDecimal;
 import java.time.ZoneId;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
@@ -80,6 +82,8 @@ import java.util.Set;
 import java.util.function.BiFunction;
 import java.util.function.Function;
 
+import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName;
+
 /** A {@link FieldMapper} for numeric types: byte, short, int, long, float and double. */
 public class NumberFieldMapper extends FieldMapper {
 
@@ -127,6 +131,7 @@ public class NumberFieldMapper extends FieldMapper {
         private final IndexVersion indexCreatedVersion;
 
         private final IndexMode indexMode;
+        private final SourceKeepMode indexSourceKeepMode;
 
         public Builder(
             String name,
@@ -134,13 +139,23 @@ public class NumberFieldMapper extends FieldMapper {
             ScriptCompiler compiler,
             Settings settings,
             IndexVersion indexCreatedVersion,
-            IndexMode mode
+            IndexMode mode,
+            SourceKeepMode indexSourceKeepMode
         ) {
-            this(name, type, compiler, IGNORE_MALFORMED_SETTING.get(settings), COERCE_SETTING.get(settings), indexCreatedVersion, mode);
+            this(
+                name,
+                type,
+                compiler,
+                IGNORE_MALFORMED_SETTING.get(settings),
+                COERCE_SETTING.get(settings),
+                indexCreatedVersion,
+                mode,
+                indexSourceKeepMode
+            );
         }
 
         public static Builder docValuesOnly(String name, NumberType type, IndexVersion indexCreatedVersion) {
-            Builder builder = new Builder(name, type, ScriptCompiler.NONE, false, false, indexCreatedVersion, null);
+            Builder builder = new Builder(name, type, ScriptCompiler.NONE, false, false, indexCreatedVersion, null, null);
             builder.indexed.setValue(false);
             builder.dimension.setValue(false);
             return builder;
@@ -153,7 +168,8 @@ public class NumberFieldMapper extends FieldMapper {
             boolean ignoreMalformedByDefault,
             boolean coerceByDefault,
             IndexVersion indexCreatedVersion,
-            IndexMode mode
+            IndexMode mode,
+            SourceKeepMode indexSourceKeepMode
         ) {
             super(name);
             this.type = type;
@@ -209,6 +225,8 @@ public class NumberFieldMapper extends FieldMapper {
 
             this.script.precludesParameters(ignoreMalformed, coerce, nullValue);
             addScriptValidation(script, indexed, hasDocValues);
+
+            this.indexSourceKeepMode = indexSourceKeepMode;
         }
 
         Builder nullValue(Number number) {
@@ -272,7 +290,16 @@ public class NumberFieldMapper extends FieldMapper {
             MappedFieldType ft = new NumberFieldType(context.buildFullName(leafName()), this, context.isSourceSynthetic());
             hasScript = script.get() != null;
             onScriptError = onScriptErrorParam.getValue();
-            return new NumberFieldMapper(leafName(), ft, builderParams(this, context), context.isSourceSynthetic(), this);
+            String offsetsFieldName = getOffsetsFieldName(
+                context,
+                indexSourceKeepMode,
+                hasDocValues.getValue(),
+                stored.getValue(),
+                this,
+                indexCreatedVersion,
+                IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_NUMBER
+            );
+            return new NumberFieldMapper(leafName(), ft, builderParams(this, context), context.isSourceSynthetic(), this, offsetsFieldName);
         }
     }
 
@@ -445,13 +472,8 @@ public class NumberFieldMapper extends FieldMapper {
             }
 
             @Override
-            SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) {
-                return new SortedNumericDocValuesSyntheticFieldLoader(fieldName, fieldSimpleName, ignoreMalformed) {
-                    @Override
-                    protected void writeValue(XContentBuilder b, long value) throws IOException {
-                        b.value(HalfFloatPoint.sortableShortToHalfFloat((short) value));
-                    }
-                };
+            public void writeValue(XContentBuilder b, long value) throws IOException {
+                b.value(HalfFloatPoint.sortableShortToHalfFloat((short) value));
             }
 
             @Override
@@ -634,13 +656,8 @@ public class NumberFieldMapper extends FieldMapper {
             }
 
             @Override
-            SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) {
-                return new SortedNumericDocValuesSyntheticFieldLoader(fieldName, fieldSimpleName, ignoreMalformed) {
-                    @Override
-                    protected void writeValue(XContentBuilder b, long value) throws IOException {
-                        b.value(NumericUtils.sortableIntToFloat((int) value));
-                    }
-                };
+            public void writeValue(XContentBuilder b, long value) throws IOException {
+                b.value(NumericUtils.sortableIntToFloat((int) value));
             }
 
             @Override
@@ -789,13 +806,8 @@ public class NumberFieldMapper extends FieldMapper {
             }
 
             @Override
-            SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) {
-                return new SortedNumericDocValuesSyntheticFieldLoader(fieldName, fieldSimpleName, ignoreMalformed) {
-                    @Override
-                    protected void writeValue(XContentBuilder b, long value) throws IOException {
-                        b.value(NumericUtils.sortableLongToDouble(value));
-                    }
-                };
+            public void writeValue(XContentBuilder b, long value) throws IOException {
+                b.value(NumericUtils.sortableLongToDouble(value));
             }
 
             @Override
@@ -838,12 +850,12 @@ public class NumberFieldMapper extends FieldMapper {
             }
 
             @Override
-            public Short parse(XContentParser parser, boolean coerce) throws IOException {
+            public Byte parse(XContentParser parser, boolean coerce) throws IOException {
                 int value = parser.intValue(coerce);
                 if (value < Byte.MIN_VALUE || value > Byte.MAX_VALUE) {
                     throw new IllegalArgumentException("Value [" + value + "] is out of range for a byte");
                 }
-                return (short) value;
+                return (byte) value;
             }
 
             @Override
@@ -912,8 +924,8 @@ public class NumberFieldMapper extends FieldMapper {
             }
 
             @Override
-            SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) {
-                return NumberType.syntheticLongFieldLoader(fieldName, fieldSimpleName, ignoreMalformed);
+            public void writeValue(XContentBuilder b, long value) throws IOException {
+                b.value(value);
             }
 
             @Override
@@ -1030,8 +1042,8 @@ public class NumberFieldMapper extends FieldMapper {
             }
 
             @Override
-            SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) {
-                return NumberType.syntheticLongFieldLoader(fieldName, fieldSimpleName, ignoreMalformed);
+            public void writeValue(XContentBuilder b, long value) throws IOException {
+                b.value(value);
             }
 
             @Override
@@ -1222,8 +1234,8 @@ public class NumberFieldMapper extends FieldMapper {
             }
 
             @Override
-            SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) {
-                return NumberType.syntheticLongFieldLoader(fieldName, fieldSimpleName, ignoreMalformed);
+            public void writeValue(XContentBuilder b, long value) throws IOException {
+                b.value(value);
             }
 
             @Override
@@ -1374,8 +1386,8 @@ public class NumberFieldMapper extends FieldMapper {
             }
 
             @Override
-            SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) {
-                return syntheticLongFieldLoader(fieldName, fieldSimpleName, ignoreMalformed);
+            public void writeValue(XContentBuilder b, long value) throws IOException {
+                b.value(value);
             }
 
             @Override
@@ -1427,7 +1439,15 @@ public class NumberFieldMapper extends FieldMapper {
             this.name = name;
             this.numericType = numericType;
             this.parser = createTypeParserWithLegacySupport(
-                (n, c) -> new Builder(n, this, c.scriptCompiler(), c.getSettings(), c.indexVersionCreated(), c.getIndexSettings().getMode())
+                (n, c) -> new Builder(
+                    n,
+                    this,
+                    c.scriptCompiler(),
+                    c.getSettings(),
+                    c.indexVersionCreated(),
+                    c.getIndexSettings().getMode(),
+                    c.getIndexSettings().sourceKeepMode()
+                )
             );
         }
 
@@ -1658,17 +1678,13 @@ public class NumberFieldMapper extends FieldMapper {
             return ((Number) value).doubleValue();
         }
 
-        abstract SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed);
+        abstract void writeValue(XContentBuilder builder, long longValue) throws IOException;
 
-        private static SourceLoader.SyntheticFieldLoader syntheticLongFieldLoader(
-            String fieldName,
-            String fieldSimpleName,
-            boolean ignoreMalformed
-        ) {
+        SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) {
             return new SortedNumericDocValuesSyntheticFieldLoader(fieldName, fieldSimpleName, ignoreMalformed) {
                 @Override
-                protected void writeValue(XContentBuilder b, long value) throws IOException {
-                    b.value(value);
+                public void writeValue(XContentBuilder b, long value) throws IOException {
+                    NumberType.this.writeValue(b, value);
                 }
             };
         }
@@ -2036,15 +2052,18 @@ public class NumberFieldMapper extends FieldMapper {
     private boolean allowMultipleValues;
     private final IndexVersion indexCreatedVersion;
     private final boolean isSyntheticSource;
+    private final String offsetsFieldName;
 
     private final IndexMode indexMode;
+    private final SourceKeepMode indexSourceKeepMode;
 
     private NumberFieldMapper(
         String simpleName,
         MappedFieldType mappedFieldType,
         BuilderParams builderParams,
         boolean isSyntheticSource,
-        Builder builder
+        Builder builder,
+        String offsetsFieldName
     ) {
         super(simpleName, mappedFieldType, builderParams);
         this.type = builder.type;
@@ -2065,6 +2084,8 @@ public class NumberFieldMapper extends FieldMapper {
         this.indexCreatedVersion = builder.indexCreatedVersion;
         this.isSyntheticSource = isSyntheticSource;
         this.indexMode = builder.indexMode;
+        this.offsetsFieldName = offsetsFieldName;
+        this.indexSourceKeepMode = builder.indexSourceKeepMode;
     }
 
     boolean coerce() {
@@ -2081,6 +2102,11 @@ public class NumberFieldMapper extends FieldMapper {
         return (NumberFieldType) super.fieldType();
     }
 
+    @Override
+    public String getOffsetFieldName() {
+        return offsetsFieldName;
+    }
+
     public NumberType type() {
         return type;
     }
@@ -2109,7 +2135,17 @@ public class NumberFieldMapper extends FieldMapper {
         }
         if (value != null) {
             indexValue(context, value);
+        } else {
+            value = fieldType().nullValue;
+        }
+        if (offsetsFieldName != null && context.isImmediateParentAnArray() && context.canAddIgnoredField()) {
+            if (value != null) {
+                context.getOffSetContext().recordOffset(offsetsFieldName, (Comparable<?>) value);
+            } else {
+                context.getOffSetContext().recordNull(offsetsFieldName);
+            }
         }
+
     }
 
     /**
@@ -2170,11 +2206,16 @@ public class NumberFieldMapper extends FieldMapper {
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new Builder(leafName(), type, scriptCompiler, ignoreMalformedByDefault, coerceByDefault, indexCreatedVersion, indexMode)
-            .dimension(dimension)
-            .metric(metricType)
-            .allowMultipleValues(allowMultipleValues)
-            .init(this);
+        return new Builder(
+            leafName(),
+            type,
+            scriptCompiler,
+            ignoreMalformedByDefault,
+            coerceByDefault,
+            indexCreatedVersion,
+            indexMode,
+            indexSourceKeepMode
+        ).dimension(dimension).metric(metricType).allowMultipleValues(allowMultipleValues).init(this);
     }
 
     @Override
@@ -2186,10 +2227,23 @@ public class NumberFieldMapper extends FieldMapper {
         }
     }
 
+    private SourceLoader.SyntheticFieldLoader docValuesSyntheticFieldLoader() {
+        if (offsetsFieldName != null) {
+            var layers = new ArrayList<CompositeSyntheticFieldLoader.Layer>();
+            layers.add(new SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer(fullPath(), offsetsFieldName, type::writeValue));
+            if (ignoreMalformed.value()) {
+                layers.add(new CompositeSyntheticFieldLoader.MalformedValuesLayer(fullPath()));
+            }
+            return new CompositeSyntheticFieldLoader(leafName(), fullPath(), layers);
+        } else {
+            return type.syntheticFieldLoader(fullPath(), leafName(), ignoreMalformed.value());
+        }
+    }
+
     @Override
     protected SyntheticSourceSupport syntheticSourceSupport() {
         if (hasDocValues) {
-            return new SyntheticSourceSupport.Native(() -> type.syntheticFieldLoader(fullPath(), leafName(), ignoreMalformed.value()));
+            return new SyntheticSourceSupport.Native(this::docValuesSyntheticFieldLoader);
         }
 
         return super.syntheticSourceSupport();

server/src/main/java/org/elasticsearch/index/mapper/SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer.java

@@ -0,0 +1,177 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.elasticsearch.common.io.stream.ByteArrayStreamInput;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+
+class SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer implements CompositeSyntheticFieldLoader.DocValuesLayer {
+    @FunctionalInterface
+    interface NumericValueWriter {
+        void writeLongValue(XContentBuilder b, long value) throws IOException;
+    }
+
+    private final String fullPath;
+    private final String offsetsFieldName;
+    private final NumericValueWriter valueWriter;
+    private NumericDocValuesWithOffsetsLoader docValuesLoader;
+
+    SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer(String fullPath, String offsetsFieldName, NumericValueWriter valueWriter) {
+        this.fullPath = fullPath;
+        this.offsetsFieldName = offsetsFieldName;
+        this.valueWriter = valueWriter;
+    }
+
+    @Override
+    public DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf) throws IOException {
+        SortedNumericDocValues valueDocValues = DocValues.getSortedNumeric(leafReader, fullPath);
+        SortedDocValues offsetDocValues = DocValues.getSorted(leafReader, offsetsFieldName);
+
+        return docValuesLoader = new NumericDocValuesWithOffsetsLoader(valueDocValues, offsetDocValues, valueWriter);
+    }
+
+    @Override
+    public boolean hasValue() {
+        return docValuesLoader != null && docValuesLoader.hasValue();
+    }
+
+    @Override
+    public long valueCount() {
+        if (docValuesLoader != null) {
+            return docValuesLoader.count();
+        } else {
+            return 0;
+        }
+    }
+
+    @Override
+    public void write(XContentBuilder b) throws IOException {
+        if (docValuesLoader != null) {
+            docValuesLoader.write(b);
+        }
+    }
+
+    @Override
+    public String fieldName() {
+        return fullPath;
+    }
+
+    private static final class NumericDocValuesWithOffsetsLoader implements DocValuesLoader {
+        private final SortedDocValues offsetDocValues;
+        private final SortedNumericDocValues valueDocValues;
+        private final NumericValueWriter writer;
+        private final ByteArrayStreamInput scratch = new ByteArrayStreamInput();
+
+        private boolean hasValue;
+        private boolean hasOffset;
+        private int[] offsetToOrd;
+
+        NumericDocValuesWithOffsetsLoader(
+            SortedNumericDocValues valueDocValues,
+            SortedDocValues offsetDocValues,
+            NumericValueWriter writer
+        ) {
+            this.valueDocValues = valueDocValues;
+            this.offsetDocValues = offsetDocValues;
+            this.writer = writer;
+        }
+
+        @Override
+        public boolean advanceToDoc(int docId) throws IOException {
+            hasValue = valueDocValues.advanceExact(docId);
+            hasOffset = offsetDocValues.advanceExact(docId);
+            if (hasValue || hasOffset) {
+                if (hasOffset) {
+                    int offsetOrd = offsetDocValues.ordValue();
+                    var encodedValue = offsetDocValues.lookupOrd(offsetOrd);
+                    scratch.reset(encodedValue.bytes, encodedValue.offset, encodedValue.length);
+                    offsetToOrd = FieldArrayContext.parseOffsetArray(scratch);
+                } else {
+                    offsetToOrd = null;
+                }
+                return true;
+            } else {
+                offsetToOrd = null;
+                return false;
+            }
+        }
+
+        public boolean hasValue() {
+            return hasOffset || (hasValue && valueDocValues.docValueCount() > 0);
+        }
+
+        public int count() {
+            if (hasValue) {
+                if (offsetToOrd != null) {
+                    // Even though there may only be one value, the fact that offsets were recorded means that
+                    // the value was in an array, so we need to trick CompositeSyntheticFieldLoader into
+                    // always serializing this layer as an array
+                    return offsetToOrd.length + 1;
+                } else {
+                    return valueDocValues.docValueCount();
+                }
+            } else {
+                if (hasOffset) {
+                    // same as above, even though there are no values, the presence of recorded offsets means
+                    // there was an array containing zero or more null values in the original source
+                    return 2;
+                } else {
+                    return 0;
+                }
+            }
+        }
+
+        public void write(XContentBuilder b) throws IOException {
+            if (hasValue == false && hasOffset == false) {
+                return;
+            }
+
+            if (offsetToOrd != null && hasValue) {
+                int count = valueDocValues.docValueCount();
+                long[] values = new long[count];
+                int duplicates = 0;
+                for (int i = 0; i < count; i++) {
+                    long value = valueDocValues.nextValue();
+                    if (i > 0 && value == values[i - duplicates - 1]) {
+                        duplicates++;
+                        continue;
+                    }
+
+                    values[i - duplicates] = value;
+                }
+
+                for (int offset : offsetToOrd) {
+                    if (offset == -1) {
+                        b.nullValue();
+                    } else {
+                        writer.writeLongValue(b, values[offset]);
+                    }
+                }
+            } else if (offsetToOrd != null) {
+                // in cased all values are NULLs
+                for (int offset : offsetToOrd) {
+                    assert offset == -1;
+                    b.nullValue();
+                }
+            } else {
+                for (int i = 0; i < valueDocValues.docValueCount(); i++) {
+                    writer.writeLongValue(b, valueDocValues.nextValue());
+                }
+            }
+        }
+
+    }
+}

server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java

@@ -106,6 +106,7 @@ public abstract class AbstractFieldDataTestCase extends ESSingleNodeTestCase {
                 false,
                 true,
                 IndexVersion.current(),
+                null,
                 null
             ).docValues(docValues).build(context).fieldType();
         } else if (type.equals("double")) {
@@ -116,6 +117,7 @@ public abstract class AbstractFieldDataTestCase extends ESSingleNodeTestCase {
                 false,
                 true,
                 IndexVersion.current(),
+                null,
                 null
             ).docValues(docValues).build(context).fieldType();
         } else if (type.equals("long")) {
@@ -126,6 +128,7 @@ public abstract class AbstractFieldDataTestCase extends ESSingleNodeTestCase {
                 false,
                 true,
                 IndexVersion.current(),
+                null,
                 null
             ).docValues(docValues).build(context).fieldType();
         } else if (type.equals("int")) {
@@ -136,6 +139,7 @@ public abstract class AbstractFieldDataTestCase extends ESSingleNodeTestCase {
                 false,
                 true,
                 IndexVersion.current(),
+                null,
                 null
             ).docValues(docValues).build(context).fieldType();
         } else if (type.equals("short")) {
@@ -146,6 +150,7 @@ public abstract class AbstractFieldDataTestCase extends ESSingleNodeTestCase {
                 false,
                 true,
                 IndexVersion.current(),
+                null,
                 null
             ).docValues(docValues).build(context).fieldType();
         } else if (type.equals("byte")) {
@@ -156,6 +161,7 @@ public abstract class AbstractFieldDataTestCase extends ESSingleNodeTestCase {
                 false,
                 true,
                 IndexVersion.current(),
+                null,
                 null
             ).docValues(docValues).build(context).fieldType();
         } else if (type.equals("geo_point")) {

server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java

@@ -85,13 +85,14 @@ public class IndexFieldDataServiceTests extends ESSingleNodeTestCase {
         assertTrue(fd instanceof SortedSetOrdinalsIndexFieldData);
 
         for (MappedFieldType mapper : Arrays.asList(
-            new NumberFieldMapper.Builder("int", BYTE, ScriptCompiler.NONE, false, true, IndexVersion.current(), null).build(context)
+            new NumberFieldMapper.Builder("int", BYTE, ScriptCompiler.NONE, false, true, IndexVersion.current(), null, null).build(context)
                 .fieldType(),
-            new NumberFieldMapper.Builder("int", SHORT, ScriptCompiler.NONE, false, true, IndexVersion.current(), null).build(context)
+            new NumberFieldMapper.Builder("int", SHORT, ScriptCompiler.NONE, false, true, IndexVersion.current(), null, null).build(context)
                 .fieldType(),
-            new NumberFieldMapper.Builder("int", INTEGER, ScriptCompiler.NONE, false, true, IndexVersion.current(), null).build(context)
-                .fieldType(),
-            new NumberFieldMapper.Builder("long", LONG, ScriptCompiler.NONE, false, true, IndexVersion.current(), null).build(context)
+            new NumberFieldMapper.Builder("int", INTEGER, ScriptCompiler.NONE, false, true, IndexVersion.current(), null, null).build(
+                context
+            ).fieldType(),
+            new NumberFieldMapper.Builder("long", LONG, ScriptCompiler.NONE, false, true, IndexVersion.current(), null, null).build(context)
                 .fieldType()
         )) {
             ifdService.clear();
@@ -106,6 +107,7 @@ public class IndexFieldDataServiceTests extends ESSingleNodeTestCase {
             false,
             true,
             IndexVersion.current(),
+            null,
             null
         ).build(context).fieldType();
         ifdService.clear();
@@ -119,6 +121,7 @@ public class IndexFieldDataServiceTests extends ESSingleNodeTestCase {
             false,
             true,
             IndexVersion.current(),
+            null,
             null
         ).build(context).fieldType();
         ifdService.clear();

server/src/test/java/org/elasticsearch/index/mapper/ByteOffsetDocValuesLoaderTests.java

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.mapper;
+
+public class ByteOffsetDocValuesLoaderTests extends OffsetDocValuesLoaderTestCase {
+    @Override
+    protected String getFieldTypeName() {
+        return "byte";
+    }
+
+    @Override
+    protected Object randomValue() {
+        return randomByte();
+    }
+}

server/src/test/java/org/elasticsearch/index/mapper/ByteSyntheticSourceNativeArrayIntegrationTests.java

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.mapper;
+
+import com.carrotsearch.randomizedtesting.generators.RandomStrings;
+
+public class ByteSyntheticSourceNativeArrayIntegrationTests extends NativeArrayIntegrationTestCase {
+
+    @Override
+    protected String getFieldTypeName() {
+        return "byte";
+    }
+
+    @Override
+    protected Byte getRandomValue() {
+        return randomByte();
+    }
+
+    @Override
+    protected String getMalformedValue() {
+        return RandomStrings.randomAsciiOfLength(random(), 8);
+    }
+}

server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java

@@ -571,7 +571,7 @@ public class DateFieldMapperTests extends MapperTestCase {
     }
 
     @Override
-    protected SyntheticSourceSupport syntheticSourceSupportForKeepTests(boolean ignoreMalformed) {
+    protected SyntheticSourceSupport syntheticSourceSupportForKeepTests(boolean ignoreMalformed, Mapper.SourceKeepMode sourceKeepMode) {
         // Serializing and deserializing BigDecimal values may lead to parsing errors, a test artifact.
         return syntheticSourceSupportInternal(ignoreMalformed, false);
     }

server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldMapperTests.java

@@ -155,4 +155,8 @@ public class DoubleFieldMapperTests extends NumberFieldMapperTests {
     protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
         return new NumberSyntheticSourceSupport(Number::doubleValue, ignoreMalformed);
     }
+
+    protected SyntheticSourceSupport syntheticSourceSupportForKeepTests(boolean ignoreMalformed, Mapper.SourceKeepMode sourceKeepMode) {
+        return new NumberSyntheticSourceSupportForKeepTests(Number::doubleValue, ignoreMalformed, sourceKeepMode);
+    }
 }

server/src/test/java/org/elasticsearch/index/mapper/DoubleOffsetDocValuesLoaderTests.java

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.mapper;
+
+public class DoubleOffsetDocValuesLoaderTests extends OffsetDocValuesLoaderTestCase {
+
+    @Override
+    protected String getFieldTypeName() {
+        return "double";
+    }
+
+    @Override
+    protected Object randomValue() {
+        return randomDouble();
+    }
+}

server/src/test/java/org/elasticsearch/index/mapper/DoubleSyntheticSourceNativeArrayIntegrationTests.java

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.mapper;
+
+import com.carrotsearch.randomizedtesting.generators.RandomStrings;
+
+public class DoubleSyntheticSourceNativeArrayIntegrationTests extends NativeArrayIntegrationTestCase {
+
+    @Override
+    protected String getFieldTypeName() {
+        return "double";
+    }
+
+    @Override
+    protected Double getRandomValue() {
+        return randomDouble();
+    }
+
+    @Override
+    protected String getMalformedValue() {
+        return RandomStrings.randomAsciiOfLength(random(), 8);
+    }
+}

server/src/test/java/org/elasticsearch/index/mapper/FloatFieldMapperTests.java

@@ -55,6 +55,12 @@ public class FloatFieldMapperTests extends NumberFieldMapperTests {
         return new NumberSyntheticSourceSupport(Number::floatValue, ignoreMalformed);
     }
 
+    @Override
+    protected SyntheticSourceSupport syntheticSourceSupportForKeepTests(boolean ignoreMalformed, Mapper.SourceKeepMode sourceKeepMode) {
+        return new NumberSyntheticSourceSupportForKeepTests(Number::floatValue, ignoreMalformed, sourceKeepMode);
+
+    }
+
     @Override
     protected Function<Object, Object> loadBlockExpected() {
         return v -> {

server/src/test/java/org/elasticsearch/index/mapper/FloatOffsetDocValuesLoaderTests.java

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.mapper;
+
+public class FloatOffsetDocValuesLoaderTests extends OffsetDocValuesLoaderTestCase {
+
+    @Override
+    protected String getFieldTypeName() {
+        return "float";
+    }
+
+    @Override
+    protected Object randomValue() {
+        return randomFloat();
+    }
+}

server/src/test/java/org/elasticsearch/index/mapper/FloatSyntheticSourceNativeArrayIntegrationTests.java

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.mapper;
+
+import com.carrotsearch.randomizedtesting.generators.RandomStrings;
+
+public class FloatSyntheticSourceNativeArrayIntegrationTests extends NativeArrayIntegrationTestCase {
+
+    @Override
+    protected String getFieldTypeName() {
+        return "float";
+    }
+
+    @Override
+    protected Float getRandomValue() {
+        return randomFloat();
+    }
+
+    @Override
+    protected String getMalformedValue() {
+        return RandomStrings.randomAsciiOfLength(random(), 8);
+    }
+}

server/src/test/java/org/elasticsearch/index/mapper/HalfFloatFieldMapperTests.java

@@ -55,6 +55,16 @@ public class HalfFloatFieldMapperTests extends NumberFieldMapperTests {
         );
     }
 
+    @Override
+    protected SyntheticSourceSupport syntheticSourceSupportForKeepTests(boolean ignoreMalformed, Mapper.SourceKeepMode sourceKeepMode) {
+        return new NumberSyntheticSourceSupportForKeepTests(
+            n -> HalfFloatPoint.sortableShortToHalfFloat(HalfFloatPoint.halfFloatToSortableShort(n.floatValue())),
+            ignoreMalformed,
+            sourceKeepMode
+        );
+
+    }
+
     @Override
     protected Function<Object, Object> loadBlockExpected() {
         return v -> {

server/src/test/java/org/elasticsearch/index/mapper/HalfFloatOffsetDocValuesLoaderTests.java

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.apache.lucene.sandbox.document.HalfFloatPoint;
+
+public class HalfFloatOffsetDocValuesLoaderTests extends OffsetDocValuesLoaderTestCase {
+
+    @Override
+    public String getFieldTypeName() {
+        return "half_float";
+    }
+
+    @Override
+    public Object randomValue() {
+        return HalfFloatPoint.sortableShortToHalfFloat(HalfFloatPoint.halfFloatToSortableShort(randomFloat()));
+    }
+}

server/src/test/java/org/elasticsearch/index/mapper/HalfFloatSyntheticSourceNativeArrayIntegrationTests.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.mapper;
+
+import com.carrotsearch.randomizedtesting.generators.RandomStrings;
+
+import org.apache.lucene.sandbox.document.HalfFloatPoint;
+
+public class HalfFloatSyntheticSourceNativeArrayIntegrationTests extends NativeArrayIntegrationTestCase {
+
+    @Override
+    protected String getFieldTypeName() {
+        return "half_float";
+    }
+
+    @Override
+    protected Float getRandomValue() {
+        return HalfFloatPoint.sortableShortToHalfFloat(HalfFloatPoint.halfFloatToSortableShort(randomFloat()));
+    }
+
+    @Override
+    protected String getMalformedValue() {
+        return RandomStrings.randomAsciiOfLength(random(), 8);
+    }
+}

server/src/test/java/org/elasticsearch/index/mapper/IPSyntheticSourceNativeArrayIntegrationTests.java

@@ -9,6 +9,8 @@
 
 package org.elasticsearch.index.mapper;
 
+import com.carrotsearch.randomizedtesting.generators.RandomStrings;
+
 import org.elasticsearch.common.network.NetworkAddress;
 
 import java.util.ArrayList;
@@ -28,6 +30,11 @@ public class IPSyntheticSourceNativeArrayIntegrationTests extends NativeArrayInt
         return NetworkAddress.format(randomIp(true));
     }
 
+    @Override
+    protected String getMalformedValue() {
+        return RandomStrings.randomAsciiOfLength(random(), 8);
+    }
+
     public void testSynthesizeArray() throws Exception {
         var arrayValues = new Object[][] {
             new Object[] { "192.168.1.4", "192.168.1.3", null, "192.168.1.2", null, "192.168.1.1" },

server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java

@@ -17,6 +17,7 @@ import org.elasticsearch.search.lookup.SourceFilter;
 import org.elasticsearch.test.FieldMaskingReader;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.hamcrest.Matchers;
+import org.junit.Before;
 
 import java.io.IOException;
 import java.math.BigInteger;
@@ -25,9 +26,9 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
+import java.util.TreeSet;
 
 public class IgnoredSourceFieldMapperTests extends MapperServiceTestCase {
-
     private DocumentMapper getDocumentMapperWithFieldLimit() throws IOException {
         return createMapperService(
             Settings.builder()
@@ -743,7 +744,7 @@ public class IgnoredSourceFieldMapperTests extends MapperServiceTestCase {
             b.startObject("int_value").field("type", "integer").endObject();
         })).documentMapper();
         var syntheticSource = syntheticSource(documentMapper, b -> b.array("int_value", new int[] { 10 }));
-        assertEquals("{\"int_value\":10}", syntheticSource);
+        assertEquals("{\"int_value\":[10]}", syntheticSource);
         ParsedDocument doc = documentMapper.parse(source(syntheticSource));
         assertNull(doc.rootDoc().getField("_ignored_source"));
     }
@@ -757,6 +758,8 @@ public class IgnoredSourceFieldMapperTests extends MapperServiceTestCase {
     }
 
     public void testIndexStoredArraySourceSingleLeafElementInObjectArray() throws IOException {
+        roundtripMaskedFields.add("path.int_value.offsets");
+
         DocumentMapper documentMapper = createMapperServiceWithStoredArraySource(mapping(b -> {
             b.startObject("path").field("synthetic_source_keep", "none").startObject("properties");
             {
@@ -843,6 +846,8 @@ public class IgnoredSourceFieldMapperTests extends MapperServiceTestCase {
     }
 
     public void testIndexStoredArraySourceRootObjectArrayWithBypass() throws IOException {
+        roundtripMaskedFields.add("path.int_value.offsets");
+
         DocumentMapper documentMapper = createMapperServiceWithStoredArraySource(mapping(b -> {
             b.startObject("path");
             {
@@ -895,6 +900,8 @@ public class IgnoredSourceFieldMapperTests extends MapperServiceTestCase {
     }
 
     public void testIndexStoredArraySourceNestedValueArrayDisabled() throws IOException {
+        roundtripMaskedFields.add("path.obj.foo.offsets");
+
         DocumentMapper documentMapper = createMapperServiceWithStoredArraySource(mapping(b -> {
             b.startObject("path");
             {
@@ -1125,6 +1132,8 @@ public class IgnoredSourceFieldMapperTests extends MapperServiceTestCase {
     }
 
     public void testConflictingFieldNameAfterArray() throws IOException {
+        roundtripMaskedFields.add("path.to.id.offsets");
+
         DocumentMapper documentMapper = createSytheticSourceMapperService(mapping(b -> {
             b.startObject("path").startObject("properties");
             {
@@ -2448,6 +2457,15 @@ public class IgnoredSourceFieldMapperTests extends MapperServiceTestCase {
         assertEquals("{\"top\":{\"level1\":{\"level2\":{\"n\":25}}}}", syntheticSource);
     }
 
+    private Set<String> roundtripMaskedFields;
+
+    @Before
+    public void resetRoundtripMaskedFields() {
+        roundtripMaskedFields = new TreeSet<>(
+            Set.of(SourceFieldMapper.RECOVERY_SOURCE_NAME, IgnoredSourceFieldMapper.NAME, SourceFieldMapper.RECOVERY_SOURCE_SIZE_NAME)
+        );
+    }
+
     protected void validateRoundTripReader(String syntheticSource, DirectoryReader reader, DirectoryReader roundTripReader)
         throws IOException {
         // We exclude ignored source field since in some cases it contains an exact copy of a part of document source.
@@ -2455,14 +2473,8 @@ public class IgnoredSourceFieldMapperTests extends MapperServiceTestCase {
         // and since the copy is exact, contents of ignored source are different.
         assertReaderEquals(
             "round trip " + syntheticSource,
-            new FieldMaskingReader(
-                Set.of(SourceFieldMapper.RECOVERY_SOURCE_NAME, IgnoredSourceFieldMapper.NAME, SourceFieldMapper.RECOVERY_SOURCE_SIZE_NAME),
-                reader
-            ),
-            new FieldMaskingReader(
-                Set.of(SourceFieldMapper.RECOVERY_SOURCE_NAME, IgnoredSourceFieldMapper.NAME, SourceFieldMapper.RECOVERY_SOURCE_SIZE_NAME),
-                roundTripReader
-            )
+            new FieldMaskingReader(roundtripMaskedFields, reader),
+            new FieldMaskingReader(roundtripMaskedFields, roundTripReader)
         );
     }
 }

server/src/test/java/org/elasticsearch/index/mapper/IntegerOffsetDocValuesLoaderTests.java

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.mapper;
+
+public class IntegerOffsetDocValuesLoaderTests extends OffsetDocValuesLoaderTestCase {
+    @Override
+    protected String getFieldTypeName() {
+        return "integer";
+    }
+
+    @Override
+    protected Object randomValue() {
+        return randomInt();
+    }
+}

server/src/test/java/org/elasticsearch/index/mapper/IntegerSyntheticSourceNativeArrayIntegrationTests.java

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.mapper;
+
+import com.carrotsearch.randomizedtesting.generators.RandomStrings;
+
+public class IntegerSyntheticSourceNativeArrayIntegrationTests extends NativeArrayIntegrationTestCase {
+
+    @Override
+    protected String getFieldTypeName() {
+        return "integer";
+    }
+
+    @Override
+    protected Integer getRandomValue() {
+        return randomInt();
+    }
+
+    @Override
+    protected String getMalformedValue() {
+        return RandomStrings.randomAsciiOfLength(random(), 8);
+    }
+}

server/src/test/java/org/elasticsearch/index/mapper/IpOffsetDocValuesLoaderTests.java

@@ -35,7 +35,7 @@ public class IpOffsetDocValuesLoaderTests extends OffsetDocValuesLoaderTestCase
     }
 
     @Override
-    protected String randomValue() {
+    protected Object randomValue() {
         return NetworkAddress.format(randomIp(true));
     }
 }

server/src/test/java/org/elasticsearch/index/mapper/KeywordOffsetDocValuesLoaderTests.java

@@ -30,7 +30,7 @@ public class KeywordOffsetDocValuesLoaderTests extends OffsetDocValuesLoaderTest
     }
 
     @Override
-    protected String randomValue() {
+    protected Object randomValue() {
         return randomAlphanumericOfLength(2);
     }
 }

server/src/test/java/org/elasticsearch/index/mapper/KeywordSyntheticSourceNativeArrayIntegrationTests.java

@@ -28,6 +28,11 @@ public class KeywordSyntheticSourceNativeArrayIntegrationTests extends NativeArr
         return RandomStrings.randomAsciiOfLength(random(), 8);
     }
 
+    @Override
+    public Object getMalformedValue() {
+        return null;
+    }
+
     public void testSynthesizeArray() throws Exception {
         var arrayValues = new Object[][] {
             new Object[] { "z", "y", null, "x", null, "v" },

server/src/test/java/org/elasticsearch/index/mapper/LongOffsetDocValuesLoaderTests.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.mapper;
+
+public class LongOffsetDocValuesLoaderTests extends OffsetDocValuesLoaderTestCase {
+
+    public void testOffsetArray() throws Exception {
+        verifyOffsets("{\"field\":[26,24,25,3,2,1]}");
+        verifyOffsets("{\"field\":[26,null,25,3,null,1]}");
+        verifyOffsets("{\"field\":[5,5,6,-3,-9,-9,5,2,5,6,-3,-9]}");
+    }
+
+    public void testOffsetNestedArray() throws Exception {
+        verifyOffsets("{\"field\":[\"26\",[\"24\"],[\"3\"],null,\"1\"]}", "{\"field\":[26,24,3,null,1]}");
+        verifyOffsets("{\"field\":[\"26\",[\"24\", [\"11\"]],[\"3\", [\"12\"]],null,\"1\"]}", "{\"field\":[26,24,11,3,12,null,1]}");
+    }
+
+    @Override
+    protected String getFieldTypeName() {
+        return "long";
+    }
+
+    @Override
+    protected Object randomValue() {
+        return randomLong();
+    }
+}

server/src/test/java/org/elasticsearch/index/mapper/LongSyntheticSourceNativeArrayIntegrationTests.java

@@ -0,0 +1,75 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.mapper;
+
+import com.carrotsearch.randomizedtesting.generators.RandomStrings;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class LongSyntheticSourceNativeArrayIntegrationTests extends NativeArrayIntegrationTestCase {
+
+    @Override
+    protected String getFieldTypeName() {
+        return "long";
+    }
+
+    @Override
+    public Long getRandomValue() {
+        return randomLong();
+    }
+
+    @Override
+    public String getMalformedValue() {
+        return RandomStrings.randomAsciiOfLength(random(), 8);
+    }
+
+    public void testSynthesizeArray() throws Exception {
+        var arrayValues = new Object[][] {
+            new Object[] { 26, null, 25, null, 24, null, 22 },
+            new Object[] { null, 2, null, -1 },
+            new Object[] { null },
+            new Object[] { null, null, null },
+            new Object[] { 3, 2, 1 },
+            new Object[] { 1 },
+            new Object[] { 3, 3, 3, 3, 3 } };
+        verifySyntheticArray(arrayValues);
+    }
+
+    public void testSynthesizeObjectArray() throws Exception {
+        List<List<Object[]>> documents = new ArrayList<>();
+        {
+            List<Object[]> document = new ArrayList<>();
+            document.add(new Object[] { 26, 25, 24 });
+            document.add(new Object[] { 13, 12, 13 });
+            document.add(new Object[] { 3, 2, 1 });
+            documents.add(document);
+        }
+        {
+            List<Object[]> document = new ArrayList<>();
+            document.add(new Object[] { -12, 14, 6 });
+            document.add(new Object[] { 1 });
+            document.add(new Object[] { -200, 4 });
+            documents.add(document);
+        }
+        verifySyntheticObjectArray(documents);
+    }
+
+    public void testSynthesizeArrayInObjectField() throws Exception {
+        List<Object[]> documents = new ArrayList<>();
+        documents.add(new Object[] { 26, 25, 24 });
+        documents.add(new Object[] { 13, 12, 13 });
+        documents.add(new Object[] { 3, 2, 1 });
+        documents.add(new Object[] { -20, 5, 7 });
+        documents.add(new Object[] { 8, 8, 8 });
+        documents.add(new Object[] { 7, 6, 5 });
+        verifySyntheticArrayInObject(documents);
+    }
+}

server/src/test/java/org/elasticsearch/index/mapper/NativeArrayIntegrationTestCase.java

@@ -15,28 +15,26 @@ import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.LeafReader;
 import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
+import org.elasticsearch.action.admin.indices.refresh.RefreshAction;
+import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.WriteRequest;
-import org.elasticsearch.common.network.NetworkAddress;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.query.IdsQueryBuilder;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.xcontent.XContentBuilder;
-import org.hamcrest.Matchers;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.LinkedHashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
 import static org.hamcrest.Matchers.contains;
-import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.nullValue;
 
 public abstract class NativeArrayIntegrationTestCase extends ESSingleNodeTestCase {
@@ -49,7 +47,7 @@ public abstract class NativeArrayIntegrationTestCase extends ESSingleNodeTestCas
     public void testSynthesizeArrayRandom() throws Exception {
         var arrayValues = new Object[randomInt(64)];
         for (int j = 0; j < arrayValues.length; j++) {
-            arrayValues[j] = NetworkAddress.format(randomIp(true));
+            arrayValues[j] = getRandomValue();
         }
         verifySyntheticArray(new Object[][] { arrayValues });
     }
@@ -67,9 +65,187 @@ public abstract class NativeArrayIntegrationTestCase extends ESSingleNodeTestCas
         verifySyntheticArrayInObject(documents);
     }
 
+    public void testSynthesizeArrayRandomIgnoresMalformed() throws Exception {
+        assumeTrue("supports ignore_malformed", getMalformedValue() != null);
+        int numDocs = randomIntBetween(8, 256);
+        List<XContentBuilder> expectedDocuments = new ArrayList<>(numDocs);
+        List<XContentBuilder> inputDocuments = new ArrayList<>(numDocs);
+        for (int i = 0; i < numDocs; i++) {
+            Object[] values = new Object[randomInt(64)];
+            Object[] malformed = new Object[randomInt(64)];
+            for (int j = 0; j < values.length; j++) {
+                values[j] = getRandomValue();
+            }
+            for (int j = 0; j < malformed.length; j++) {
+                malformed[j] = getMalformedValue();
+            }
+
+            var expectedDocument = jsonBuilder().startObject();
+            var inputDocument = jsonBuilder().startObject();
+
+            boolean expectedContainsArray = values.length > 0 || malformed.length > 1;
+            if (expectedContainsArray) {
+                expectedDocument.startArray("field");
+            } else if (malformed.length > 0) {
+                expectedDocument.field("field");
+            }
+            inputDocument.startArray("field");
+
+            int valuesIdx = 0;
+            int malformedIdx = 0;
+            for (int j = 0; j < values.length + malformed.length; j++) {
+                if (j < values.length) {
+                    expectedDocument.value(values[j]);
+                } else {
+                    expectedDocument.value(malformed[j - values.length]);
+                }
+
+                if (valuesIdx == values.length) {
+                    inputDocument.value(malformed[malformedIdx++]);
+                } else if (malformedIdx == malformed.length) {
+                    inputDocument.value(values[valuesIdx++]);
+                } else {
+                    if (randomBoolean()) {
+                        inputDocument.value(values[valuesIdx++]);
+                    } else {
+                        inputDocument.value(malformed[malformedIdx++]);
+                    }
+                }
+            }
+
+            if (expectedContainsArray) {
+                expectedDocument.endArray();
+            }
+            expectedDocument.endObject();
+            inputDocument.endArray().endObject();
+
+            expectedDocuments.add(expectedDocument);
+            inputDocuments.add(inputDocument);
+        }
+
+        var mapping = jsonBuilder().startObject()
+            .startObject("properties")
+            .startObject("field")
+            .field("type", getFieldTypeName())
+            .field("ignore_malformed", true)
+            .endObject()
+            .endObject()
+            .endObject();
+        var indexService = createIndex(
+            "test-index",
+            Settings.builder().put("index.mapping.source.mode", "synthetic").put("index.mapping.synthetic_source_keep", "arrays").build(),
+            mapping
+        );
+        for (int i = 0; i < inputDocuments.size(); i++) {
+            var document = inputDocuments.get(i);
+            var indexRequest = new IndexRequest("test-index");
+            indexRequest.id("my-id-" + i);
+
+            indexRequest.source(document);
+            client().index(indexRequest).actionGet();
+        }
+
+        var refreshRequest = new RefreshRequest("test-index");
+        client().execute(RefreshAction.INSTANCE, refreshRequest).actionGet();
+
+        for (int i = 0; i < expectedDocuments.size(); i++) {
+            var document = expectedDocuments.get(i);
+            String expectedSource = Strings.toString(document);
+            var searchRequest = new SearchRequest("test-index");
+            searchRequest.source().query(new IdsQueryBuilder().addIds("my-id-" + i));
+            var searchResponse = client().search(searchRequest).actionGet();
+            try {
+                var hit = searchResponse.getHits().getHits()[0];
+                assertThat(hit.getId(), equalTo("my-id-" + i));
+                assertThat(hit.getSourceAsString(), equalTo(expectedSource));
+            } finally {
+                searchResponse.decRef();
+            }
+        }
+    }
+
+    public void testSynthesizeRandomArrayInNestedContext() throws Exception {
+        var arrayValues = new Object[randomIntBetween(1, 8)][randomIntBetween(2, 64)];
+        for (int i = 0; i < arrayValues.length; i++) {
+            for (int j = 0; j < arrayValues[i].length; j++) {
+                arrayValues[i][j] = randomInt(10) == 0 ? null : getRandomValue();
+            }
+        }
+
+        var mapping = jsonBuilder().startObject()
+            .startObject("properties")
+            .startObject("parent")
+            .field("type", "nested")
+            .startObject("properties")
+            .startObject("field")
+            .field("type", getFieldTypeName())
+            .endObject()
+            .endObject()
+            .endObject()
+            .endObject()
+            .endObject();
+
+        var indexService = createIndex(
+            "test-index",
+            Settings.builder().put("index.mapping.source.mode", "synthetic").put("index.mapping.synthetic_source_keep", "arrays").build(),
+            mapping
+        );
+
+        var indexRequest = new IndexRequest("test-index");
+        indexRequest.id("my-id-1");
+        var source = jsonBuilder().startObject().startArray("parent");
+        for (Object[] arrayValue : arrayValues) {
+            source.startObject().array("field", arrayValue).endObject();
+        }
+        source.endArray().endObject();
+        indexRequest.source(source);
+        indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
+        client().index(indexRequest).actionGet();
+
+        var expectedSource = jsonBuilder().startObject();
+        if (arrayValues.length > 1) {
+            expectedSource.startArray("parent");
+        } else {
+            expectedSource.field("parent");
+        }
+        for (Object[] arrayValue : arrayValues) {
+            expectedSource.startObject();
+            expectedSource.array("field", arrayValue);
+            expectedSource.endObject();
+        }
+        if (arrayValues.length > 1) {
+            expectedSource.endArray();
+        }
+        expectedSource.endObject();
+        var expected = Strings.toString(expectedSource);
+
+        var searchRequest = new SearchRequest("test-index");
+        searchRequest.source().query(new IdsQueryBuilder().addIds("my-id-1"));
+        var searchResponse = client().search(searchRequest).actionGet();
+        try {
+            var hit = searchResponse.getHits().getHits()[0];
+            assertThat(hit.getId(), equalTo("my-id-1"));
+            assertThat(hit.getSourceAsString(), equalTo(expected));
+        } finally {
+            searchResponse.decRef();
+        }
+
+        assertThat(indexService.mapperService().mappingLookup().getMapper("parent.field").getOffsetFieldName(), nullValue());
+
+        try (var searcher = indexService.getShard(0).acquireSearcher(getTestName())) {
+            var reader = searcher.getDirectoryReader();
+            var document = reader.storedFields().document(0);
+            Set<String> storedFieldNames = new LinkedHashSet<>(document.getFields().stream().map(IndexableField::name).toList());
+            assertThat(storedFieldNames, contains("_ignored_source"));
+            assertThat(FieldInfos.getMergedFieldInfos(reader).fieldInfo("parent.field.offsets"), nullValue());
+        }
+    }
+
     protected abstract String getFieldTypeName();
 
-    protected abstract String getRandomValue();
+    protected abstract Object getRandomValue();
+
+    protected abstract Object getMalformedValue();
 
     protected void verifySyntheticArray(Object[][] arrays) throws IOException {
         var mapping = jsonBuilder().startObject()
@@ -103,7 +279,9 @@ public abstract class NativeArrayIntegrationTestCase extends ESSingleNodeTestCas
             } else {
                 source.field("field").nullValue();
             }
-            indexRequest.source(source.endObject());
+            source.endObject();
+            var expectedSource = Strings.toString(source);
+            indexRequest.source(source);
             indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
             client().index(indexRequest).actionGet();
 
@@ -113,16 +291,7 @@ public abstract class NativeArrayIntegrationTestCase extends ESSingleNodeTestCas
             try {
                 var hit = searchResponse.getHits().getHits()[0];
                 assertThat(hit.getId(), equalTo("my-id-" + i));
-                var sourceAsMap = hit.getSourceAsMap();
-                assertThat(sourceAsMap, hasKey("field"));
-                var actualArray = (List<?>) sourceAsMap.get("field");
-                if (array == null) {
-                    assertThat(actualArray, nullValue());
-                } else if (array.length == 0) {
-                    assertThat(actualArray, empty());
-                } else {
-                    assertThat(actualArray, Matchers.contains(array));
-                }
+                assertThat(hit.getSourceAsString(), equalTo(expectedSource));
             } finally {
                 searchResponse.decRef();
             }
@@ -169,8 +338,9 @@ public abstract class NativeArrayIntegrationTestCase extends ESSingleNodeTestCas
                 source.array("field", arrayValue);
                 source.endObject();
             }
-            source.endArray();
-            indexRequest.source(source.endObject());
+            source.endArray().endObject();
+            var expectedSource = Strings.toString(source);
+            indexRequest.source(source);
             indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
             client().index(indexRequest).actionGet();
 
@@ -180,13 +350,7 @@ public abstract class NativeArrayIntegrationTestCase extends ESSingleNodeTestCas
             try {
                 var hit = searchResponse.getHits().getHits()[0];
                 assertThat(hit.getId(), equalTo("my-id-" + i));
-                var sourceAsMap = hit.getSourceAsMap();
-                var objectArray = (List<?>) sourceAsMap.get("object");
-                for (int j = 0; j < document.size(); j++) {
-                    var expected = document.get(j);
-                    List<?> actual = (List<?>) ((Map<?, ?>) objectArray.get(j)).get("field");
-                    assertThat(actual, Matchers.contains(expected));
-                }
+                assertThat(hit.getSourceAsString(), equalTo(expectedSource));
             } finally {
                 searchResponse.decRef();
             }
@@ -223,7 +387,7 @@ public abstract class NativeArrayIntegrationTestCase extends ESSingleNodeTestCas
                 .startObject("object")
                 .startObject("properties")
                 .startObject("field")
-                .field("type", "keyword")
+                .field("type", getFieldTypeName())
                 .endObject()
                 .endObject()
                 .endObject()
@@ -238,8 +402,9 @@ public abstract class NativeArrayIntegrationTestCase extends ESSingleNodeTestCas
             var source = jsonBuilder().startObject();
             source.startObject("object");
             source.array("field", arrayValue);
-            source.endObject();
-            indexRequest.source(source.endObject());
+            source.endObject().endObject();
+            var expectedSource = Strings.toString(source);
+            indexRequest.source(source);
             indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
             client().index(indexRequest).actionGet();
 
@@ -249,17 +414,7 @@ public abstract class NativeArrayIntegrationTestCase extends ESSingleNodeTestCas
             try {
                 var hit = searchResponse.getHits().getHits()[0];
                 assertThat(hit.getId(), equalTo("my-id-" + i));
-                var sourceAsMap = hit.getSourceAsMap();
-                var objectArray = (Map<?, ?>) sourceAsMap.get("object");
-
-                List<?> actual = (List<?>) objectArray.get("field");
-                if (arrayValue == null) {
-                    assertThat(actual, nullValue());
-                } else if (arrayValue.length == 0) {
-                    assertThat(actual, empty());
-                } else {
-                    assertThat(actual, Matchers.contains(arrayValue));
-                }
+                assertThat(hit.getSourceAsString(), equalTo(expectedSource));
             } finally {
                 searchResponse.decRef();
             }

server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java

@@ -943,6 +943,7 @@ public class NumberFieldTypeTests extends FieldTypeTestCase {
             false,
             true,
             IndexVersion.current(),
+            null,
             null
         ).build(MapperBuilderContext.root(false, false)).fieldType();
         assertEquals(List.of(3), fetchSourceValue(mapper, 3.14));
@@ -956,6 +957,7 @@ public class NumberFieldTypeTests extends FieldTypeTestCase {
             false,
             true,
             IndexVersion.current(),
+            null,
             null
         ).nullValue(2.71f).build(MapperBuilderContext.root(false, false)).fieldType();
         assertEquals(List.of(2.71f), fetchSourceValue(nullValueMapper, ""));
@@ -970,6 +972,7 @@ public class NumberFieldTypeTests extends FieldTypeTestCase {
             false,
             true,
             IndexVersion.current(),
+            null,
             null
         ).build(MapperBuilderContext.root(false, false)).fieldType();
         /*

server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java

@@ -336,6 +336,7 @@ public final class ObjectMapperMergeTests extends ESTestCase {
                 false,
                 true,
                 IndexVersion.current(),
+                null,
                 null
             )
         ).build(MapperBuilderContext.root(false, false));

server/src/test/java/org/elasticsearch/index/mapper/OffsetDocValuesLoaderTestCase.java

@@ -17,6 +17,7 @@ import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentType;
 
 import java.io.IOException;
+import java.util.HashSet;
 
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
 import static org.hamcrest.Matchers.nullValue;
@@ -159,25 +160,32 @@ public abstract class OffsetDocValuesLoaderTestCase extends MapperServiceTestCas
     }
 
     public void testOffsetArrayRandom() throws Exception {
-        StringBuilder values = new StringBuilder();
+        String values;
         int numValues = randomIntBetween(0, 256);
-        for (int i = 0; i < numValues; i++) {
-            if (randomInt(10) == 1) {
-                values.append("null");
-            } else {
-                String randomValue = randomValue();
-                values.append('"').append(randomValue).append('"');
-            }
-            if (i != (numValues - 1)) {
-                values.append(',');
+
+        var previousValues = new HashSet<>();
+        try (XContentBuilder b = XContentBuilder.builder(XContentType.JSON.xContent());) {
+            b.startArray();
+            for (int i = 0; i < numValues; i++) {
+                if (randomInt(10) == 1) {
+                    b.nullValue();
+                } else if (randomInt(10) == 1 && previousValues.size() > 0) {
+                    b.value(randomFrom(previousValues));
+                } else {
+                    Object value = randomValue();
+                    previousValues.add(value);
+                    b.value(value);
+                }
             }
+            b.endArray();
+            values = Strings.toString(b);
         }
-        verifyOffsets("{\"field\":[" + values + "]}");
+        verifyOffsets("{\"field\":" + values + "}");
     }
 
     protected abstract String getFieldTypeName();
 
-    protected abstract String randomValue();
+    protected abstract Object randomValue();
 
     protected void verifyOffsets(String source) throws IOException {
         verifyOffsets(source, source);

server/src/test/java/org/elasticsearch/index/mapper/ShortOffsetDocValuesLoaderTests.java

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.mapper;
+
+public class ShortOffsetDocValuesLoaderTests extends OffsetDocValuesLoaderTestCase {
+    @Override
+    protected String getFieldTypeName() {
+        return "short";
+    }
+
+    @Override
+    protected Object randomValue() {
+        return randomShort();
+    }
+}

server/src/test/java/org/elasticsearch/index/mapper/ShortSyntheticSourceNativeArrayIntegrationTests.java

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.mapper;
+
+import com.carrotsearch.randomizedtesting.generators.RandomStrings;
+
+public class ShortSyntheticSourceNativeArrayIntegrationTests extends NativeArrayIntegrationTestCase {
+
+    @Override
+    protected String getFieldTypeName() {
+        return "short";
+    }
+
+    @Override
+    protected Short getRandomValue() {
+        return randomShort();
+    }
+
+    @Override
+    protected String getMalformedValue() {
+        return RandomStrings.randomAsciiOfLength(random(), 8);
+    }
+}

test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java

@@ -1671,12 +1671,14 @@ public abstract class MapperTestCase extends MapperServiceTestCase {
         }), equalTo("{}"));
     }
 
-    protected SyntheticSourceSupport syntheticSourceSupportForKeepTests(boolean ignoreMalformed) {
+    protected SyntheticSourceSupport syntheticSourceSupportForKeepTests(boolean ignoreMalformed, Mapper.SourceKeepMode keepMode) {
         return syntheticSourceSupport(ignoreMalformed);
     }
 
     public void testSyntheticSourceKeepNone() throws IOException {
-        SyntheticSourceExample example = syntheticSourceSupportForKeepTests(shouldUseIgnoreMalformed()).example(1);
+        SyntheticSourceExample example = syntheticSourceSupportForKeepTests(shouldUseIgnoreMalformed(), Mapper.SourceKeepMode.NONE).example(
+            1
+        );
         DocumentMapper mapper = createSytheticSourceMapperService(mapping(b -> {
             b.startObject("field");
             b.field("synthetic_source_keep", "none");
@@ -1687,7 +1689,9 @@ public abstract class MapperTestCase extends MapperServiceTestCase {
     }
 
     public void testSyntheticSourceKeepAll() throws IOException {
-        SyntheticSourceExample example = syntheticSourceSupportForKeepTests(shouldUseIgnoreMalformed()).example(1);
+        SyntheticSourceExample example = syntheticSourceSupportForKeepTests(shouldUseIgnoreMalformed(), Mapper.SourceKeepMode.ALL).example(
+            1
+        );
         DocumentMapper mapperAll = createSytheticSourceMapperService(mapping(b -> {
             b.startObject("field");
             b.field("synthetic_source_keep", "all");
@@ -1704,7 +1708,8 @@ public abstract class MapperTestCase extends MapperServiceTestCase {
     }
 
     public void testSyntheticSourceKeepArrays() throws IOException {
-        SyntheticSourceExample example = syntheticSourceSupportForKeepTests(shouldUseIgnoreMalformed()).example(1);
+        SyntheticSourceExample example = syntheticSourceSupportForKeepTests(shouldUseIgnoreMalformed(), Mapper.SourceKeepMode.ARRAYS)
+            .example(1);
         DocumentMapper mapperAll = createSytheticSourceMapperService(mapping(b -> {
             b.startObject("field");
             b.field("synthetic_source_keep", randomSyntheticSourceKeep());

test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java

@@ -396,7 +396,36 @@ public abstract class NumberFieldMapperTests extends MapperTestCase {
 
     protected abstract Number randomNumber();
 
-    protected final class NumberSyntheticSourceSupport implements SyntheticSourceSupport {
+    protected final class NumberSyntheticSourceSupportForKeepTests extends NumberSyntheticSourceSupport {
+        private final boolean preserveSource;
+
+        protected NumberSyntheticSourceSupportForKeepTests(
+            Function<Number, Number> round,
+            boolean ignoreMalformed,
+            Mapper.SourceKeepMode sourceKeepMode
+        ) {
+            super(round, ignoreMalformed);
+            this.preserveSource = sourceKeepMode == Mapper.SourceKeepMode.ALL;
+        }
+
+        @Override
+        public boolean preservesExactSource() {
+            return preserveSource;
+        }
+
+        @Override
+        public SyntheticSourceExample example(int maxVals) {
+            var example = super.example(maxVals);
+            return new SyntheticSourceExample(
+                example.expectedForSyntheticSource(),
+                example.expectedForSyntheticSource(),
+                example.expectedForBlockLoader(),
+                example.mapping()
+            );
+        }
+    }
+
+    protected class NumberSyntheticSourceSupport implements SyntheticSourceSupport {
         private final Long nullValue = usually() ? null : randomNumber().longValue();
         private final boolean coerce = rarely();
         private final boolean docValues = randomBoolean();

test/framework/src/main/java/org/elasticsearch/index/mapper/WholeNumberFieldMapperTests.java

@@ -123,4 +123,9 @@ public abstract class WholeNumberFieldMapperTests extends NumberFieldMapperTests
     protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
         return new NumberSyntheticSourceSupport(Number::longValue, ignoreMalformed);
     }
+
+    @Override
+    protected SyntheticSourceSupport syntheticSourceSupportForKeepTests(boolean ignoreMalformed, Mapper.SourceKeepMode sourceKeepMode) {
+        return new NumberSyntheticSourceSupportForKeepTests(Number::longValue, ignoreMalformed, sourceKeepMode);
+    }
 }

x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldMapper.java

@@ -168,8 +168,15 @@ public class AggregateMetricDoubleFieldMapper extends FieldMapper {
 
         private final IndexVersion indexCreatedVersion;
         private final IndexMode indexMode;
-
-        public Builder(String name, Boolean ignoreMalformedByDefault, IndexVersion indexCreatedVersion, IndexMode mode) {
+        private final SourceKeepMode indexSourceKeepMode;
+
+        public Builder(
+            String name,
+            Boolean ignoreMalformedByDefault,
+            IndexVersion indexCreatedVersion,
+            IndexMode mode,
+            SourceKeepMode indexSourceKeepMode
+        ) {
             super(name);
             this.ignoreMalformed = Parameter.boolParam(
                 Names.IGNORE_MALFORMED,
@@ -181,6 +188,7 @@ public class AggregateMetricDoubleFieldMapper extends FieldMapper {
             this.timeSeriesMetric = TimeSeriesParams.metricParam(m -> toType(m).metricType, MetricType.GAUGE);
             this.indexCreatedVersion = Objects.requireNonNull(indexCreatedVersion);
             this.indexMode = mode;
+            this.indexSourceKeepMode = indexSourceKeepMode;
         }
 
         @Override
@@ -238,7 +246,8 @@ public class AggregateMetricDoubleFieldMapper extends FieldMapper {
                         false,
                         false,
                         indexCreatedVersion,
-                        indexMode
+                        indexMode,
+                        indexSourceKeepMode
                     ).allowMultipleValues(false);
                 } else {
                     builder = new NumberFieldMapper.Builder(
@@ -248,7 +257,8 @@ public class AggregateMetricDoubleFieldMapper extends FieldMapper {
                         false,
                         true,
                         indexCreatedVersion,
-                        indexMode
+                        indexMode,
+                        indexSourceKeepMode
                     ).allowMultipleValues(false);
                 }
                 NumberFieldMapper fieldMapper = builder.build(context);
@@ -274,7 +284,13 @@ public class AggregateMetricDoubleFieldMapper extends FieldMapper {
     }
 
     public static final FieldMapper.TypeParser PARSER = new TypeParser(
-        (n, c) -> new Builder(n, IGNORE_MALFORMED_SETTING.get(c.getSettings()), c.indexVersionCreated(), c.getIndexSettings().getMode()),
+        (n, c) -> new Builder(
+            n,
+            IGNORE_MALFORMED_SETTING.get(c.getSettings()),
+            c.indexVersionCreated(),
+            c.getIndexSettings().getMode(),
+            c.getIndexSettings().sourceKeepMode()
+        ),
         notInMultiFields(CONTENT_TYPE)
     );
 
@@ -673,6 +689,7 @@ public class AggregateMetricDoubleFieldMapper extends FieldMapper {
     private final TimeSeriesParams.MetricType metricType;
 
     private final IndexMode indexMode;
+    private final SourceKeepMode indexSourceKeepMode;
 
     private AggregateMetricDoubleFieldMapper(
         String simpleName,
@@ -690,6 +707,7 @@ public class AggregateMetricDoubleFieldMapper extends FieldMapper {
         this.metricType = builder.timeSeriesMetric.getValue();
         this.indexCreatedVersion = builder.indexCreatedVersion;
         this.indexMode = builder.indexMode;
+        this.indexSourceKeepMode = builder.indexSourceKeepMode;
     }
 
     @Override
@@ -842,7 +860,8 @@ public class AggregateMetricDoubleFieldMapper extends FieldMapper {
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new Builder(leafName(), ignoreMalformedByDefault, indexCreatedVersion, indexMode).metric(metricType).init(this);
+        return new Builder(leafName(), ignoreMalformedByDefault, indexCreatedVersion, indexMode, indexSourceKeepMode).metric(metricType)
+            .init(this);
     }
 
     @Override

x-pack/plugin/mapper-counted-keyword/src/test/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapperTests.java

@@ -16,6 +16,7 @@ import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperTestCase;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.plugins.Plugin;
@@ -118,7 +119,8 @@ public class CountedKeywordFieldMapperTests extends MapperTestCase {
     }
 
     public void testSyntheticSourceIndexLevelKeepArrays() throws IOException {
-        SyntheticSourceExample example = syntheticSourceSupportForKeepTests(shouldUseIgnoreMalformed()).example(1);
+        SyntheticSourceExample example = syntheticSourceSupportForKeepTests(shouldUseIgnoreMalformed(), Mapper.SourceKeepMode.ARRAYS)
+            .example(1);
         XContentBuilder mappings = mapping(b -> {
             b.startObject("field");
             example.mapping().accept(b);

x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java

@@ -16,6 +16,7 @@ import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.DocumentParsingException;
 import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.NumberTypeOutOfRangeSpec;
@@ -382,6 +383,11 @@ public class UnsignedLongFieldMapperTests extends WholeNumberFieldMapperTests {
         return new NumberSyntheticSourceSupport(ignoreMalformed);
     }
 
+    @Override
+    protected SyntheticSourceSupport syntheticSourceSupportForKeepTests(boolean ignoreMalformed, Mapper.SourceKeepMode sourceKeepMode) {
+        return syntheticSourceSupport(ignoreMalformed);
+    }
+
     @Override
     protected IngestScriptSupport ingestScriptSupport() {
         throw new AssumptionViolatedException("not supported");

x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java

@@ -720,6 +720,7 @@ public class RollupIndexerIndexingTests extends AggregatorTestCase {
                     false,
                     false,
                     IndexVersion.current(),
+                    null,
                     null
                 ).build(MapperBuilderContext.root(false, false)).fieldType();
                 fieldTypes.put(ft.name(), ft);
@@ -744,6 +745,7 @@ public class RollupIndexerIndexingTests extends AggregatorTestCase {
                     false,
                     false,
                     IndexVersion.current(),
+                    null,
                     null
                 ).build(MapperBuilderContext.root(false, false)).fieldType();
                 fieldTypes.put(ft.name(), ft);