Use FallbackSyntheticSourceBlockLoader for number fields (#122280) (#122452)

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java

@@ -261,7 +261,8 @@ public class ValuesSourceReaderBenchmark {
             null,
             false,
             null,
-            null
+            null,
+            false
         ).blockLoader(null);
     }
 

benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java

@@ -83,7 +83,7 @@ public class ScriptScoreBenchmark {
     private final ScriptModule scriptModule = new ScriptModule(Settings.EMPTY, pluginsService.filterPlugins(ScriptPlugin.class).toList());
 
     private final Map<String, MappedFieldType> fieldTypes = Map.ofEntries(
-        Map.entry("n", new NumberFieldType("n", NumberType.LONG, false, false, true, true, null, Map.of(), null, false, null, null))
+        Map.entry("n", new NumberFieldType("n", NumberType.LONG, false, false, true, true, null, Map.of(), null, false, null, null, false))
     );
     private final IndexFieldDataCache fieldDataCache = new IndexFieldDataCache.None();
     private final CircuitBreakerService breakerService = new NoneCircuitBreakerService();

docs/changelog/122280.yaml

@@ -0,0 +1,5 @@
+pr: 122280
+summary: Use `FallbackSyntheticSourceBlockLoader` for number fields
+area: Mapping
+type: enhancement
+issues: []

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java

@@ -86,7 +86,8 @@ public class TokenCountFieldMapper extends FieldMapper {
                 store.getValue(),
                 hasDocValues.getValue(),
                 nullValue.getValue(),
-                meta.getValue()
+                meta.getValue(),
+                context.isSourceSynthetic()
             );
             return new TokenCountFieldMapper(leafName(), ft, builderParams(this, context), this);
         }
@@ -100,7 +101,8 @@ public class TokenCountFieldMapper extends FieldMapper {
             boolean isStored,
             boolean hasDocValues,
             Number nullValue,
-            Map<String, String> meta
+            Map<String, String> meta,
+            boolean isSyntheticSource
         ) {
             super(
                 name,
@@ -114,7 +116,8 @@ public class TokenCountFieldMapper extends FieldMapper {
                 null,
                 false,
                 null,
-                null
+                null,
+                isSyntheticSource
             );
         }
 

plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java

@@ -50,7 +50,7 @@ public class SizeFieldMapper extends MetadataFieldMapper {
 
     private static class SizeFieldType extends NumberFieldType {
         SizeFieldType() {
-            super(NAME, NumberType.INTEGER, true, true, true, false, null, Collections.emptyMap(), null, false, null, null);
+            super(NAME, NumberType.INTEGER, true, true, true, false, null, Collections.emptyMap(), null, false, null, null, false);
         }
 
         @Override

server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java

@@ -270,7 +270,7 @@ public class NumberFieldMapper extends FieldMapper {
                 dimension.setValue(true);
             }
 
-            MappedFieldType ft = new NumberFieldType(context.buildFullName(leafName()), this);
+            MappedFieldType ft = new NumberFieldType(context.buildFullName(leafName()), this, context.isSourceSynthetic());
             hasScript = script.get() != null;
             onScriptError = onScriptErrorParam.getValue();
             return new NumberFieldMapper(leafName(), ft, builderParams(this, context), context.isSourceSynthetic(), this);
@@ -464,6 +464,11 @@ public class NumberFieldMapper extends FieldMapper {
             BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) {
                 return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup);
             }
+
+            @Override
+            BlockLoader blockLoaderFromFallbackSyntheticSource(String fieldName, Number nullValue, boolean coerce) {
+                return floatingPointBlockLoaderFromFallbackSyntheticSource(this, fieldName, nullValue, coerce);
+            }
         },
         FLOAT("float", NumericType.FLOAT) {
             @Override
@@ -648,6 +653,11 @@ public class NumberFieldMapper extends FieldMapper {
             BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) {
                 return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup);
             }
+
+            @Override
+            BlockLoader blockLoaderFromFallbackSyntheticSource(String fieldName, Number nullValue, boolean coerce) {
+                return floatingPointBlockLoaderFromFallbackSyntheticSource(this, fieldName, nullValue, coerce);
+            }
         },
         DOUBLE("double", NumericType.DOUBLE) {
             @Override
@@ -798,6 +808,11 @@ public class NumberFieldMapper extends FieldMapper {
             BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) {
                 return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup);
             }
+
+            @Override
+            BlockLoader blockLoaderFromFallbackSyntheticSource(String fieldName, Number nullValue, boolean coerce) {
+                return floatingPointBlockLoaderFromFallbackSyntheticSource(this, fieldName, nullValue, coerce);
+            }
         },
         BYTE("byte", NumericType.BYTE) {
             @Override
@@ -912,6 +927,11 @@ public class NumberFieldMapper extends FieldMapper {
                 return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup);
             }
 
+            @Override
+            BlockLoader blockLoaderFromFallbackSyntheticSource(String fieldName, Number nullValue, boolean coerce) {
+                return integerBlockLoaderFromFallbackSyntheticSource(this, fieldName, nullValue, coerce);
+            }
+
             private boolean isOutOfRange(Object value) {
                 double doubleValue = objectToDouble(value);
                 return doubleValue < Byte.MIN_VALUE || doubleValue > Byte.MAX_VALUE;
@@ -1025,6 +1045,11 @@ public class NumberFieldMapper extends FieldMapper {
                 return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup);
             }
 
+            @Override
+            BlockLoader blockLoaderFromFallbackSyntheticSource(String fieldName, Number nullValue, boolean coerce) {
+                return integerBlockLoaderFromFallbackSyntheticSource(this, fieldName, nullValue, coerce);
+            }
+
             private boolean isOutOfRange(Object value) {
                 double doubleValue = objectToDouble(value);
                 return doubleValue < Short.MIN_VALUE || doubleValue > Short.MAX_VALUE;
@@ -1211,6 +1236,11 @@ public class NumberFieldMapper extends FieldMapper {
             BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) {
                 return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup);
             }
+
+            @Override
+            BlockLoader blockLoaderFromFallbackSyntheticSource(String fieldName, Number nullValue, boolean coerce) {
+                return integerBlockLoaderFromFallbackSyntheticSource(this, fieldName, nullValue, coerce);
+            }
         },
         LONG("long", NumericType.LONG) {
             @Override
@@ -1359,6 +1389,26 @@ public class NumberFieldMapper extends FieldMapper {
                 return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher, lookup);
             }
 
+            @Override
+            BlockLoader blockLoaderFromFallbackSyntheticSource(String fieldName, Number nullValue, boolean coerce) {
+                var reader = new NumberFallbackSyntheticSourceReader(this, nullValue, coerce) {
+                    @Override
+                    public void writeToBlock(List<Number> values, BlockLoader.Builder blockBuilder) {
+                        var builder = (BlockLoader.LongBuilder) blockBuilder;
+                        for (var value : values) {
+                            builder.appendLong(value.longValue());
+                        }
+                    }
+                };
+
+                return new FallbackSyntheticSourceBlockLoader(reader, fieldName) {
+                    @Override
+                    public Builder builder(BlockFactory factory, int expectedCount) {
+                        return factory.longs(expectedCount);
+                    }
+                };
+            }
+
             private boolean isOutOfRange(Object value) {
                 if (value instanceof Long) {
                     return false;
@@ -1635,6 +1685,106 @@ public class NumberFieldMapper extends FieldMapper {
         abstract BlockLoader blockLoaderFromDocValues(String fieldName);
 
         abstract BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup);
+
+        abstract BlockLoader blockLoaderFromFallbackSyntheticSource(String fieldName, Number nullValue, boolean coerce);
+
+        // All values that fit into integer are returned as integers
+        private static BlockLoader integerBlockLoaderFromFallbackSyntheticSource(
+            NumberType type,
+            String fieldName,
+            Number nullValue,
+            boolean coerce
+        ) {
+            var reader = new NumberFallbackSyntheticSourceReader(type, nullValue, coerce) {
+                @Override
+                public void writeToBlock(List<Number> values, BlockLoader.Builder blockBuilder) {
+                    var builder = (BlockLoader.IntBuilder) blockBuilder;
+                    for (var value : values) {
+                        builder.appendInt(value.intValue());
+                    }
+                }
+            };
+
+            return new FallbackSyntheticSourceBlockLoader(reader, fieldName) {
+                @Override
+                public Builder builder(BlockFactory factory, int expectedCount) {
+                    return factory.ints(expectedCount);
+                }
+            };
+        }
+
+        // All floating point values are returned as doubles
+        private static BlockLoader floatingPointBlockLoaderFromFallbackSyntheticSource(
+            NumberType type,
+            String fieldName,
+            Number nullValue,
+            boolean coerce
+        ) {
+            var reader = new NumberFallbackSyntheticSourceReader(type, nullValue, coerce) {
+                @Override
+                public void writeToBlock(List<Number> values, BlockLoader.Builder blockBuilder) {
+                    var builder = (BlockLoader.DoubleBuilder) blockBuilder;
+                    for (var value : values) {
+                        builder.appendDouble(value.doubleValue());
+                    }
+                }
+            };
+
+            return new FallbackSyntheticSourceBlockLoader(reader, fieldName) {
+                @Override
+                public Builder builder(BlockFactory factory, int expectedCount) {
+                    return factory.doubles(expectedCount);
+                }
+            };
+        }
+
+        abstract static class NumberFallbackSyntheticSourceReader extends FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport<
+            Number> {
+            private final NumberType type;
+            private final Number nullValue;
+            private final boolean coerce;
+
+            NumberFallbackSyntheticSourceReader(NumberType type, Number nullValue, boolean coerce) {
+                super(nullValue);
+                this.type = type;
+                this.nullValue = nullValue;
+                this.coerce = coerce;
+            }
+
+            @Override
+            public void convertValue(Object value, List<Number> accumulator) {
+                if (coerce && value.equals("")) {
+                    if (nullValue != null) {
+                        accumulator.add(nullValue);
+                    }
+                }
+
+                try {
+                    var converted = type.parse(value, coerce);
+                    accumulator.add(converted);
+                } catch (Exception e) {
+                    // Malformed value, skip it
+                }
+            }
+
+            @Override
+            public void parseNonNullValue(XContentParser parser, List<Number> accumulator) throws IOException {
+                // Aligned with implementation of `value(XContentParser)`
+                if (coerce && parser.currentToken() == Token.VALUE_STRING && parser.textLength() == 0) {
+                    if (nullValue != null) {
+                        accumulator.add(nullValue);
+                    }
+                }
+
+                try {
+                    Number rawValue = type.parse(parser, coerce);
+                    // Transform number to correct type (e.g. reduce precision)
+                    accumulator.add(type.parse(rawValue, coerce));
+                } catch (Exception e) {
+                    // Malformed value, skip it
+                }
+            }
+        };
     }
 
     public static class NumberFieldType extends SimpleMappedFieldType {
@@ -1646,6 +1796,7 @@ public class NumberFieldMapper extends FieldMapper {
         private final boolean isDimension;
         private final MetricType metricType;
         private final IndexMode indexMode;
+        private final boolean isSyntheticSource;
 
         public NumberFieldType(
             String name,
@@ -1659,7 +1810,8 @@ public class NumberFieldMapper extends FieldMapper {
             FieldValues<Number> script,
             boolean isDimension,
             MetricType metricType,
-            IndexMode indexMode
+            IndexMode indexMode,
+            boolean isSyntheticSource
         ) {
             super(name, isIndexed, isStored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_WITHOUT_TERMS, meta);
             this.type = Objects.requireNonNull(type);
@@ -1669,9 +1821,10 @@ public class NumberFieldMapper extends FieldMapper {
             this.isDimension = isDimension;
             this.metricType = metricType;
             this.indexMode = indexMode;
+            this.isSyntheticSource = isSyntheticSource;
         }
 
-        NumberFieldType(String name, Builder builder) {
+        NumberFieldType(String name, Builder builder, boolean isSyntheticSource) {
             this(
                 name,
                 builder.type,
@@ -1684,7 +1837,8 @@ public class NumberFieldMapper extends FieldMapper {
                 builder.scriptValues(),
                 builder.dimension.getValue(),
                 builder.metric.getValue(),
-                builder.indexMode
+                builder.indexMode,
+                isSyntheticSource
             );
         }
 
@@ -1693,7 +1847,7 @@ public class NumberFieldMapper extends FieldMapper {
         }
 
         public NumberFieldType(String name, NumberType type, boolean isIndexed) {
-            this(name, type, isIndexed, false, true, true, null, Collections.emptyMap(), null, false, null, null);
+            this(name, type, isIndexed, false, true, true, null, Collections.emptyMap(), null, false, null, null, false);
         }
 
         @Override
@@ -1770,6 +1924,11 @@ public class NumberFieldMapper extends FieldMapper {
             if (hasDocValues()) {
                 return type.blockLoaderFromDocValues(name());
             }
+
+            if (isSyntheticSource) {
+                return type.blockLoaderFromFallbackSyntheticSource(name(), nullValue, coerce);
+            }
+
             BlockSourceReader.LeafIteratorLookup lookup = isStored() || isIndexed()
                 ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name())
                 : BlockSourceReader.lookupMatchingAll();
@@ -1885,7 +2044,7 @@ public class NumberFieldMapper extends FieldMapper {
     private final MetricType metricType;
     private boolean allowMultipleValues;
     private final IndexVersion indexCreatedVersion;
-    private final boolean storeMalformedFields;
+    private final boolean isSyntheticSource;
 
     private final IndexMode indexMode;
 
@@ -1893,7 +2052,7 @@ public class NumberFieldMapper extends FieldMapper {
         String simpleName,
         MappedFieldType mappedFieldType,
         BuilderParams builderParams,
-        boolean storeMalformedFields,
+        boolean isSyntheticSource,
         Builder builder
     ) {
         super(simpleName, mappedFieldType, builderParams);
@@ -1913,7 +2072,7 @@ public class NumberFieldMapper extends FieldMapper {
         this.metricType = builder.metric.getValue();
         this.allowMultipleValues = builder.allowMultipleValues;
         this.indexCreatedVersion = builder.indexCreatedVersion;
-        this.storeMalformedFields = storeMalformedFields;
+        this.isSyntheticSource = isSyntheticSource;
         this.indexMode = builder.indexMode;
     }
 
@@ -1948,7 +2107,7 @@ public class NumberFieldMapper extends FieldMapper {
         } catch (IllegalArgumentException e) {
             if (ignoreMalformed.value() && context.parser().currentToken().isValue()) {
                 context.addIgnoredField(mappedFieldType.name());
-                if (storeMalformedFields) {
+                if (isSyntheticSource) {
                     // Save a copy of the field so synthetic source can load it
                     context.doc().add(IgnoreMalformedStoredValues.storedField(fullPath(), context.parser()));
                 }

server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java

@@ -333,7 +333,8 @@ public class IndexFieldDataServiceTests extends ESSingleNodeTestCase {
                 null,
                 false,
                 null,
-                null
+                null,
+                false
             )
         );
     }
@@ -353,7 +354,8 @@ public class IndexFieldDataServiceTests extends ESSingleNodeTestCase {
                 null,
                 false,
                 null,
-                null
+                null,
+                false
             )
         );
     }

server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java

@@ -140,7 +140,8 @@ public class NumberFieldTypeTests extends FieldTypeTestCase {
             null,
             false,
             null,
-            null
+            null,
+            false
         );
     }
 

server/src/test/java/org/elasticsearch/index/mapper/blockloader/ByteFieldBlockLoaderTests.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.mapper.blockloader;
+
+import org.elasticsearch.logsdb.datageneration.FieldType;
+
+public class ByteFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase<Integer> {
+    public ByteFieldBlockLoaderTests() {
+        super(FieldType.BYTE);
+    }
+
+    @Override
+    protected Integer convert(Number value) {
+        // All values that fit into int are represented as ints
+        return value.intValue();
+    }
+}

server/src/test/java/org/elasticsearch/index/mapper/blockloader/DoubleFieldBlockLoaderTests.java

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.mapper.blockloader;
+
+import org.elasticsearch.logsdb.datageneration.FieldType;
+
+public class DoubleFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase<Double> {
+    public DoubleFieldBlockLoaderTests() {
+        super(FieldType.DOUBLE);
+    }
+
+    @Override
+    protected Double convert(Number value) {
+        return value.doubleValue();
+    }
+}

server/src/test/java/org/elasticsearch/index/mapper/blockloader/FloatFieldBlockLoaderTests.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.mapper.blockloader;
+
+import org.elasticsearch.logsdb.datageneration.FieldType;
+
+public class FloatFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase<Double> {
+    public FloatFieldBlockLoaderTests() {
+        super(FieldType.FLOAT);
+    }
+
+    @Override
+    protected Double convert(Number value) {
+        // All float values are represented as double
+        return value.doubleValue();
+    }
+}

server/src/test/java/org/elasticsearch/index/mapper/blockloader/HalfFloatFieldBlockLoaderTests.java

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.mapper.blockloader;
+
+import org.apache.lucene.sandbox.document.HalfFloatPoint;
+import org.elasticsearch.logsdb.datageneration.FieldType;
+
+public class HalfFloatFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase<Double> {
+    public HalfFloatFieldBlockLoaderTests() {
+        super(FieldType.HALF_FLOAT);
+    }
+
+    @Override
+    protected Double convert(Number value) {
+        // All float values are represented as double
+        return (double) HalfFloatPoint.sortableShortToHalfFloat(HalfFloatPoint.halfFloatToSortableShort(value.floatValue()));
+    }
+}

server/src/test/java/org/elasticsearch/index/mapper/blockloader/IntegerFieldBlockLoaderTests.java

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.mapper.blockloader;
+
+import org.elasticsearch.logsdb.datageneration.FieldType;
+
+public class IntegerFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase<Integer> {
+    public IntegerFieldBlockLoaderTests() {
+        super(FieldType.INTEGER);
+    }
+
+    @Override
+    protected Integer convert(Number value) {
+        return value.intValue();
+    }
+}

server/src/test/java/org/elasticsearch/index/mapper/blockloader/KeywordFieldBlockLoaderTests.java

@@ -59,18 +59,6 @@ public class KeywordFieldBlockLoaderTests extends BlockLoaderTestCase {
         return maybeFoldList(resultList);
     }
 
-    private Object maybeFoldList(List<?> list) {
-        if (list.isEmpty()) {
-            return null;
-        }
-
-        if (list.size() == 1) {
-            return list.get(0);
-        }
-
-        return list;
-    }
-
     private BytesRef convert(String value, String nullValue, int ignoreAbove) {
         if (value == null) {
             if (nullValue != null) {

server/src/test/java/org/elasticsearch/index/mapper/blockloader/LongFieldBlockLoaderTests.java

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.mapper.blockloader;
+
+import org.elasticsearch.logsdb.datageneration.FieldType;
+
+public class LongFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase<Long> {
+    public LongFieldBlockLoaderTests() {
+        super(FieldType.LONG);
+    }
+
+    @Override
+    protected Long convert(Number value) {
+        return value.longValue();
+    }
+}

server/src/test/java/org/elasticsearch/index/mapper/blockloader/NumberFieldBlockLoaderTestCase.java

@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.mapper.blockloader;
+
+import org.elasticsearch.index.mapper.BlockLoaderTestCase;
+import org.elasticsearch.logsdb.datageneration.FieldType;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+public abstract class NumberFieldBlockLoaderTestCase<T extends Number> extends BlockLoaderTestCase {
+    public NumberFieldBlockLoaderTestCase(FieldType fieldType) {
+        super(fieldType);
+    }
+
+    @Override
+    @SuppressWarnings("unchecked")
+    protected Object expected(Map<String, Object> fieldMapping, Object value, boolean syntheticSource) {
+        var nullValue = fieldMapping.get("null_value") != null ? convert((Number) fieldMapping.get("null_value")) : null;
+
+        if (value instanceof List<?> == false) {
+            return convert(value, nullValue);
+        }
+
+        if ((boolean) fieldMapping.getOrDefault("doc_values", false)) {
+            // Sorted and no duplicates
+            var resultList = ((List<Object>) value).stream().map(v -> convert(v, nullValue)).filter(Objects::nonNull).sorted().toList();
+            return maybeFoldList(resultList);
+        }
+
+        // parsing from source
+        var resultList = ((List<Object>) value).stream().map(v -> convert(v, nullValue)).filter(Objects::nonNull).toList();
+        return maybeFoldList(resultList);
+    }
+
+    @SuppressWarnings("unchecked")
+    private T convert(Object value, T nullValue) {
+        if (value == null) {
+            return nullValue;
+        }
+        // String coercion is true by default
+        if (value instanceof String s && s.isEmpty()) {
+            return nullValue;
+        }
+        if (value instanceof Number n) {
+            return convert(n);
+        }
+
+        // Malformed values are excluded
+        return null;
+    }
+
+    protected abstract T convert(Number value);
+}

server/src/test/java/org/elasticsearch/index/mapper/blockloader/ShortFieldBlockLoaderTests.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.mapper.blockloader;
+
+import org.elasticsearch.logsdb.datageneration.FieldType;
+
+public class ShortFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase<Integer> {
+    public ShortFieldBlockLoaderTests() {
+        super(FieldType.SHORT);
+    }
+
+    @Override
+    protected Integer convert(Number value) {
+        // All values that fit into int are represented as ints
+        return value.intValue();
+    }
+}

server/src/test/java/org/elasticsearch/search/aggregations/AggregatorBaseTests.java

@@ -92,7 +92,8 @@ public class AggregatorBaseTests extends MapperServiceTestCase {
             null,
             false,
             null,
-            null
+            null,
+            false
         );
         return ValuesSourceConfig.resolveFieldOnly(ft, context);
     }

server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java

@@ -1494,7 +1494,8 @@ public class FiltersAggregatorTests extends AggregatorTestCase {
             null,
             false,
             null,
-            null
+            null,
+            false
         );
         docValuesFieldExistsTestCase(new ExistsQueryBuilder("f"), ft, true, i -> {
             final LuceneDocument document = new LuceneDocument();
@@ -1517,7 +1518,8 @@ public class FiltersAggregatorTests extends AggregatorTestCase {
                 null,
                 false,
                 null,
-                null
+                null,
+                false
             )
         );
     }

server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java

@@ -311,7 +311,8 @@ public class RangeAggregatorTests extends AggregatorTestCase {
                     null,
                     false,
                     null,
-                    null
+                    null,
+                    false
                 )
             )
         );
@@ -426,7 +427,8 @@ public class RangeAggregatorTests extends AggregatorTestCase {
             null,
             false,
             null,
-            null
+            null,
+            false
         );
 
         long start = 2L << 54; // Double stores 53 bits of mantissa, so we aggregate a bunch of bigger values
@@ -707,7 +709,8 @@ public class RangeAggregatorTests extends AggregatorTestCase {
             null,
             false,
             null,
-            null
+            null,
+            false
         );
         RangeAggregationBuilder aggregationBuilder = new RangeAggregationBuilder("test_range_agg");
         aggregationBuilder.field(NUMBER_FIELD_NAME);

server/src/test/java/org/elasticsearch/search/collapse/CollapseBuilderTests.java

@@ -154,7 +154,8 @@ public class CollapseBuilderTests extends AbstractXContentSerializingTestCase<Co
                 null,
                 false,
                 null,
-                null
+                null,
+                false
             );
             when(searchExecutionContext.getFieldType("field")).thenReturn(numberFieldType);
             IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> builder.build(searchExecutionContext));
@@ -172,7 +173,8 @@ public class CollapseBuilderTests extends AbstractXContentSerializingTestCase<Co
                 null,
                 false,
                 null,
-                null
+                null,
+                false
             );
             when(searchExecutionContext.getFieldType("field")).thenReturn(numberFieldType);
             builder.setInnerHits(new InnerHitBuilder().setName("field"));

test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java

@@ -145,6 +145,18 @@ public abstract class BlockLoaderTestCase extends MapperServiceTestCase {
         }
     }
 
+    protected static Object maybeFoldList(List<?> list) {
+        if (list.isEmpty()) {
+            return null;
+        }
+
+        if (list.size() == 1) {
+            return list.get(0);
+        }
+
+        return list;
+    }
+
     private Object setupAndInvokeBlockLoader(MapperService mapperService, XContentBuilder document, String fieldName) throws IOException {
         try (Directory directory = newDirectory()) {
             RandomIndexWriter iw = new RandomIndexWriter(random(), directory);

x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java

@@ -197,7 +197,8 @@ public class TimeSeriesRateAggregatorTests extends AggregatorTestCase {
             null,
             false,
             TimeSeriesParams.MetricType.COUNTER,
-            IndexMode.TIME_SERIES
+            IndexMode.TIME_SERIES,
+            false
         );
     }