Store arrays offsets for unsigned long fields natively with synthetic source (#125709) (#125746)

This patch builds on the work in #113757, #122999, #124594, and #125529 to
natively store array offsets for unsigned long fields instead of falling
back to ignored source when synthetic_source_keep: arrays.

(cherry picked from commit 689eaf20f4636a5805db03887ac73b683832291c)

# Conflicts:
#	server/src/main/java/org/elasticsearch/index/IndexVersions.java
#	x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java

docs/changelog/125709.yaml

@@ -0,0 +1,5 @@
+pr: 125709
+summary: Store arrays offsets for unsigned long fields natively with synthetic source
+area: Mapping
+type: enhancement
+issues: []

server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java

@@ -665,7 +665,7 @@ public class BooleanFieldMapper extends FieldMapper {
 
     private SourceLoader.SyntheticFieldLoader docValuesSyntheticFieldLoader() {
         if (offsetsFieldName != null) {
-            var layers = new ArrayList<CompositeSyntheticFieldLoader.Layer>();
+            var layers = new ArrayList<CompositeSyntheticFieldLoader.Layer>(2);
             layers.add(
                 new SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer(
                     fullPath(),

server/src/main/java/org/elasticsearch/index/mapper/FieldArrayContext.java

@@ -27,14 +27,14 @@ public class FieldArrayContext {
     private static final String OFFSETS_FIELD_NAME_SUFFIX = ".offsets";
     private final Map<String, Offsets> offsetsPerField = new HashMap<>();
 
-    void recordOffset(String field, Comparable<?> value) {
+    public void recordOffset(String field, Comparable<?> value) {
         Offsets arrayOffsets = offsetsPerField.computeIfAbsent(field, k -> new Offsets());
         int nextOffset = arrayOffsets.currentOffset++;
         var offsets = arrayOffsets.valueToOffsets.computeIfAbsent(value, s -> new ArrayList<>(2));
         offsets.add(nextOffset);
     }
 
-    void recordNull(String field) {
+    public void recordNull(String field) {
         Offsets arrayOffsets = offsetsPerField.computeIfAbsent(field, k -> new Offsets());
         int nextOffset = arrayOffsets.currentOffset++;
         arrayOffsets.nullValueOffsets.add(nextOffset);
@@ -82,7 +82,7 @@ public class FieldArrayContext {
         return offsetToOrd;
     }
 
-    static String getOffsetsFieldName(
+    public static String getOffsetsFieldName(
         MapperBuilderContext context,
         Mapper.SourceKeepMode indexSourceKeepMode,
         boolean hasDocValues,

server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java

@@ -646,7 +646,7 @@ public class IpFieldMapper extends FieldMapper {
     protected SyntheticSourceSupport syntheticSourceSupport() {
         if (hasDocValues) {
             return new SyntheticSourceSupport.Native(() -> {
-                var layers = new ArrayList<CompositeSyntheticFieldLoader.Layer>();
+                var layers = new ArrayList<CompositeSyntheticFieldLoader.Layer>(2);
                 if (offsetsFieldName != null) {
                     layers.add(
                         new SortedSetWithOffsetsDocValuesSyntheticFieldLoaderLayer(fullPath(), offsetsFieldName, IpFieldMapper::convert)

server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java

@@ -1154,7 +1154,7 @@ public final class KeywordFieldMapper extends FieldMapper {
     public SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fullFieldName, String leafFieldName) {
         assert fieldType.stored() || hasDocValues;
 
-        var layers = new ArrayList<CompositeSyntheticFieldLoader.Layer>();
+        var layers = new ArrayList<CompositeSyntheticFieldLoader.Layer>(2);
         if (fieldType.stored()) {
             layers.add(new CompositeSyntheticFieldLoader.StoredFieldLayer(fullPath()) {
                 @Override

server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java

@@ -2275,7 +2275,7 @@ public class NumberFieldMapper extends FieldMapper {
 
     private SourceLoader.SyntheticFieldLoader docValuesSyntheticFieldLoader() {
         if (offsetsFieldName != null) {
-            var layers = new ArrayList<CompositeSyntheticFieldLoader.Layer>();
+            var layers = new ArrayList<CompositeSyntheticFieldLoader.Layer>(2);
             layers.add(new SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer(fullPath(), offsetsFieldName, type::writeValue));
             if (ignoreMalformed.value()) {
                 layers.add(new CompositeSyntheticFieldLoader.MalformedValuesLayer(fullPath()));

server/src/main/java/org/elasticsearch/index/mapper/SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer.java

@@ -18,9 +18,9 @@ import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
 
-class SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer implements CompositeSyntheticFieldLoader.DocValuesLayer {
+public class SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer implements CompositeSyntheticFieldLoader.DocValuesLayer {
     @FunctionalInterface
-    interface NumericValueWriter {
+    public interface NumericValueWriter {
         void writeLongValue(XContentBuilder b, long value) throws IOException;
     }
 
@@ -29,7 +29,11 @@ class SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer implements Comp
     private final NumericValueWriter valueWriter;
     private NumericDocValuesWithOffsetsLoader docValuesLoader;
 
-    SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer(String fullPath, String offsetsFieldName, NumericValueWriter valueWriter) {
+    public SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer(
+        String fullPath,
+        String offsetsFieldName,
+        NumericValueWriter valueWriter
+    ) {
         this.fullPath = fullPath;
         this.offsetsFieldName = offsetsFieldName;
         this.valueWriter = valueWriter;

x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java

@@ -20,6 +20,8 @@ import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.fielddata.FieldDataContext;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
@@ -27,6 +29,7 @@ import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData;
 import org.elasticsearch.index.mapper.BlockDocValuesReader;
 import org.elasticsearch.index.mapper.BlockLoader;
 import org.elasticsearch.index.mapper.BlockSourceReader;
+import org.elasticsearch.index.mapper.CompositeSyntheticFieldLoader;
 import org.elasticsearch.index.mapper.DocumentParserContext;
 import org.elasticsearch.index.mapper.FallbackSyntheticSourceBlockLoader;
 import org.elasticsearch.index.mapper.FieldMapper;
@@ -37,6 +40,8 @@ import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MappingLookup;
 import org.elasticsearch.index.mapper.SimpleMappedFieldType;
 import org.elasticsearch.index.mapper.SortedNumericDocValuesSyntheticFieldLoader;
+import org.elasticsearch.index.mapper.SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer;
+import org.elasticsearch.index.mapper.SourceLoader;
 import org.elasticsearch.index.mapper.SourceValueFetcher;
 import org.elasticsearch.index.mapper.TextSearchInfo;
 import org.elasticsearch.index.mapper.TimeSeriesParams;
@@ -64,6 +69,7 @@ import java.util.Objects;
 import java.util.Set;
 import java.util.function.Function;
 
+import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName;
 import static org.elasticsearch.xpack.unsignedlong.UnsignedLongLeafFieldData.convertUnsignedLongToDouble;
 
 public class UnsignedLongFieldMapper extends FieldMapper {
@@ -97,13 +103,27 @@ public class UnsignedLongFieldMapper extends FieldMapper {
          */
         private final Parameter<MetricType> metric;
 
+        private final IndexVersion indexCreatedVersion;
         private final IndexMode indexMode;
+        private final SourceKeepMode indexSourceKeepMode;
 
-        public Builder(String name, Settings settings, IndexMode mode) {
-            this(name, IGNORE_MALFORMED_SETTING.get(settings), mode);
+        public Builder(
+            String name,
+            Settings settings,
+            IndexVersion indexCreatedVersion,
+            IndexMode mode,
+            SourceKeepMode indexSourceKeepMode
+        ) {
+            this(name, IGNORE_MALFORMED_SETTING.get(settings), indexCreatedVersion, mode, indexSourceKeepMode);
         }
 
-        public Builder(String name, boolean ignoreMalformedByDefault, IndexMode mode) {
+        public Builder(
+            String name,
+            boolean ignoreMalformedByDefault,
+            IndexVersion indexCreatedVersion,
+            IndexMode mode,
+            SourceKeepMode indexSourceKeepMode
+        ) {
             super(name);
             this.ignoreMalformed = Parameter.explicitBoolParam(
                 "ignore_malformed",
@@ -150,6 +170,9 @@ public class UnsignedLongFieldMapper extends FieldMapper {
                     );
                 }
             }).precludesParameters(dimension);
+
+            this.indexCreatedVersion = indexCreatedVersion;
+            this.indexSourceKeepMode = indexSourceKeepMode;
         }
 
         private String parseNullValueAsString(Object o) {
@@ -211,11 +234,35 @@ public class UnsignedLongFieldMapper extends FieldMapper {
                 indexMode,
                 context.isSourceSynthetic()
             );
-            return new UnsignedLongFieldMapper(leafName(), fieldType, builderParams(this, context), context.isSourceSynthetic(), this);
+            String offsetsFieldName = getOffsetsFieldName(
+                context,
+                indexSourceKeepMode,
+                hasDocValues.getValue(),
+                stored.getValue(),
+                this,
+                indexCreatedVersion,
+                IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY
+            );
+            return new UnsignedLongFieldMapper(
+                leafName(),
+                fieldType,
+                builderParams(this, context),
+                context.isSourceSynthetic(),
+                this,
+                offsetsFieldName
+            );
         }
     }
 
-    public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.getSettings(), c.getIndexSettings().getMode()));
+    public static final TypeParser PARSER = new TypeParser(
+        (n, c) -> new Builder(
+            n,
+            c.getSettings(),
+            c.indexVersionCreated(),
+            c.getIndexSettings().getMode(),
+            c.getIndexSettings().sourceKeepMode()
+        )
+    );
 
     public static final class UnsignedLongFieldType extends SimpleMappedFieldType {
 
@@ -640,14 +687,18 @@ public class UnsignedLongFieldMapper extends FieldMapper {
     private final Long nullValueIndexed; // null value to use for indexing, represented as shifted to signed long range
     private final boolean dimension;
     private final MetricType metricType;
+    private final IndexVersion indexCreatedVersion;
     private final IndexMode indexMode;
+    private final String offsetsFieldName;
+    private final SourceKeepMode indexSourceKeepMode;
 
     private UnsignedLongFieldMapper(
         String simpleName,
         MappedFieldType mappedFieldType,
         BuilderParams builderParams,
         boolean isSourceSynthetic,
-        Builder builder
+        Builder builder,
+        String offsetsFieldName
     ) {
         super(simpleName, mappedFieldType, builderParams);
         this.isSourceSynthetic = isSourceSynthetic;
@@ -666,6 +717,9 @@ public class UnsignedLongFieldMapper extends FieldMapper {
         this.dimension = builder.dimension.getValue();
         this.metricType = builder.metric.getValue();
         this.indexMode = builder.indexMode;
+        this.indexCreatedVersion = builder.indexCreatedVersion;
+        this.offsetsFieldName = offsetsFieldName;
+        this.indexSourceKeepMode = builder.indexSourceKeepMode;
     }
 
     @Override
@@ -678,6 +732,11 @@ public class UnsignedLongFieldMapper extends FieldMapper {
         return (UnsignedLongFieldType) super.fieldType();
     }
 
+    @Override
+    public String getOffsetFieldName() {
+        return offsetsFieldName;
+    }
+
     @Override
     protected String contentType() {
         return CONTENT_TYPE;
@@ -714,7 +773,6 @@ public class UnsignedLongFieldMapper extends FieldMapper {
         boolean isNullValue = false;
         if (numericValue == null) {
             numericValue = nullValueIndexed;
-            if (numericValue == null) return;
             isNullValue = true;
         } else {
             numericValue = unsignedToSortableSignedLong(numericValue);
@@ -724,29 +782,41 @@ public class UnsignedLongFieldMapper extends FieldMapper {
             context.getRoutingFields().addUnsignedLong(fieldType().name(), numericValue);
         }
 
-        List<Field> fields = new ArrayList<>();
-        if (indexed && hasDocValues) {
-            fields.add(new LongField(fieldType().name(), numericValue));
-        } else if (hasDocValues) {
-            fields.add(new SortedNumericDocValuesField(fieldType().name(), numericValue));
-        } else if (indexed) {
-            fields.add(new LongPoint(fieldType().name(), numericValue));
-        }
-        if (stored) {
-            // for stored field, keeping original unsigned_long value in the String form
-            String storedValued = isNullValue ? nullValue : Long.toUnsignedString(unsignedToSortableSignedLong(numericValue));
-            fields.add(new StoredField(fieldType().name(), storedValued));
+        if (numericValue != null) {
+            List<Field> fields = new ArrayList<>();
+            if (indexed && hasDocValues) {
+                fields.add(new LongField(fieldType().name(), numericValue));
+            } else if (hasDocValues) {
+                fields.add(new SortedNumericDocValuesField(fieldType().name(), numericValue));
+            } else if (indexed) {
+                fields.add(new LongPoint(fieldType().name(), numericValue));
+            }
+            if (stored) {
+                // for stored field, keeping original unsigned_long value in the String form
+                String storedValued = isNullValue ? nullValue : Long.toUnsignedString(unsignedToSortableSignedLong(numericValue));
+                fields.add(new StoredField(fieldType().name(), storedValued));
+            }
+            context.doc().addAll(fields);
+
+            if (hasDocValues == false && (stored || indexed)) {
+                context.addToFieldNames(fieldType().name());
+            }
         }
-        context.doc().addAll(fields);
 
-        if (hasDocValues == false && (stored || indexed)) {
-            context.addToFieldNames(fieldType().name());
+        if (offsetsFieldName != null && context.isImmediateParentAnArray() && context.canAddIgnoredField()) {
+            if (numericValue != null) {
+                context.getOffSetContext().recordOffset(offsetsFieldName, numericValue);
+            } else {
+                context.getOffSetContext().recordNull(offsetsFieldName);
+            }
         }
     }
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new Builder(leafName(), ignoreMalformedByDefault, indexMode).dimension(dimension).metric(metricType).init(this);
+        return new Builder(leafName(), ignoreMalformedByDefault, indexCreatedVersion, indexMode, indexSourceKeepMode).dimension(dimension)
+            .metric(metricType)
+            .init(this);
     }
 
     /**
@@ -827,17 +897,34 @@ public class UnsignedLongFieldMapper extends FieldMapper {
         }
     }
 
+    private SourceLoader.SyntheticFieldLoader docValuesSyntheticFieldLoader() {
+        if (offsetsFieldName != null) {
+            var layers = new ArrayList<CompositeSyntheticFieldLoader.Layer>(2);
+            layers.add(
+                new SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer(
+                    fullPath(),
+                    offsetsFieldName,
+                    (b, value) -> b.value(DocValueFormat.UNSIGNED_LONG_SHIFTED.format(value))
+                )
+            );
+            if (ignoreMalformed.value()) {
+                layers.add(new CompositeSyntheticFieldLoader.MalformedValuesLayer(fullPath()));
+            }
+            return new CompositeSyntheticFieldLoader(leafName(), fullPath(), layers);
+        } else {
+            return new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed()) {
+                @Override
+                protected void writeValue(XContentBuilder b, long value) throws IOException {
+                    b.value(DocValueFormat.UNSIGNED_LONG_SHIFTED.format(value));
+                }
+            };
+        }
+    }
+
     @Override
     protected SyntheticSourceSupport syntheticSourceSupport() {
         if (hasDocValues) {
-            return new SyntheticSourceSupport.Native(
-                () -> new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed()) {
-                    @Override
-                    protected void writeValue(XContentBuilder b, long value) throws IOException {
-                        b.value(DocValueFormat.UNSIGNED_LONG_SHIFTED.format(value));
-                    }
-                }
-            );
+            return new SyntheticSourceSupport.Native(this::docValuesSyntheticFieldLoader);
         }
 
         return super.syntheticSourceSupport();

x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java

@@ -385,7 +385,20 @@ public class UnsignedLongFieldMapperTests extends WholeNumberFieldMapperTests {
 
     @Override
     protected SyntheticSourceSupport syntheticSourceSupportForKeepTests(boolean ignoreMalformed, Mapper.SourceKeepMode sourceKeepMode) {
-        return syntheticSourceSupport(ignoreMalformed);
+        return new NumberSyntheticSourceSupport(ignoreMalformed) {
+            @Override
+            public SyntheticSourceExample example(int maxVals) {
+                var example = super.example(maxVals);
+                // Need the expectedForSyntheticSource as inputValue since MapperTestCase#testSyntheticSourceKeepArrays
+                // uses the inputValue as both the input and expected.
+                return new SyntheticSourceExample(
+                    example.expectedForSyntheticSource(),
+                    example.expectedForSyntheticSource(),
+                    example.expectedForBlockLoader(),
+                    example.mapping()
+                );
+            }
+        };
     }
 
     @Override
@@ -437,7 +450,7 @@ public class UnsignedLongFieldMapperTests extends WholeNumberFieldMapperTests {
         };
     }
 
-    final class NumberSyntheticSourceSupport implements SyntheticSourceSupport {
+    class NumberSyntheticSourceSupport implements SyntheticSourceSupport {
         private final BigInteger nullValue = usually() ? null : BigInteger.valueOf(randomNonNegativeLong());
         private final boolean ignoreMalformedEnabled;
 

x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java

@@ -169,16 +169,17 @@ public class UnsignedLongFieldTypeTests extends FieldTypeTestCase {
     }
 
     public void testFetchSourceValue() throws IOException {
-        MappedFieldType mapper = new UnsignedLongFieldMapper.Builder("field", false, null).build(MapperBuilderContext.root(false, false))
-            .fieldType();
+        MappedFieldType mapper = new UnsignedLongFieldMapper.Builder("field", false, null, null, null).build(
+            MapperBuilderContext.root(false, false)
+        ).fieldType();
         assertEquals(List.of(0L), fetchSourceValue(mapper, 0L));
         assertEquals(List.of(9223372036854775807L), fetchSourceValue(mapper, 9223372036854775807L));
         assertEquals(List.of(BIGINTEGER_2_64_MINUS_ONE), fetchSourceValue(mapper, "18446744073709551615"));
         assertEquals(List.of(), fetchSourceValue(mapper, ""));
 
-        MappedFieldType nullValueMapper = new UnsignedLongFieldMapper.Builder("field", false, null).nullValue("18446744073709551615")
-            .build(MapperBuilderContext.root(false, false))
-            .fieldType();
+        MappedFieldType nullValueMapper = new UnsignedLongFieldMapper.Builder("field", false, null, null, null).nullValue(
+            "18446744073709551615"
+        ).build(MapperBuilderContext.root(false, false)).fieldType();
         assertEquals(List.of(BIGINTEGER_2_64_MINUS_ONE), fetchSourceValue(nullValueMapper, ""));
     }
 }

x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongOffsetDocValuesLoaderTests.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.unsignedlong;
+
+import org.elasticsearch.index.mapper.OffsetDocValuesLoaderTestCase;
+import org.elasticsearch.plugins.Plugin;
+
+import java.util.Collection;
+import java.util.List;
+
+public class UnsignedLongOffsetDocValuesLoaderTests extends OffsetDocValuesLoaderTestCase {
+
+    @Override
+    protected Collection<? extends Plugin> getPlugins() {
+        return List.of(new UnsignedLongMapperPlugin());
+    }
+
+    @Override
+    public String getFieldTypeName() {
+        return "unsigned_long";
+    }
+
+    @Override
+    public Object randomValue() {
+        return randomNonNegativeLong();
+    }
+}

x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongSyntheticSourceNativeArrayIntegrationTests.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.unsignedlong;
+
+import com.carrotsearch.randomizedtesting.generators.RandomStrings;
+
+import org.elasticsearch.index.mapper.NativeArrayIntegrationTestCase;
+import org.elasticsearch.plugins.Plugin;
+
+import java.util.Collection;
+import java.util.List;
+
+public class UnsignedLongSyntheticSourceNativeArrayIntegrationTests extends NativeArrayIntegrationTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return List.of(UnsignedLongMapperPlugin.class);
+    }
+
+    @Override
+    protected String getFieldTypeName() {
+        return "unsigned_long";
+    }
+
+    @Override
+    protected Long getRandomValue() {
+        return randomNonNegativeLong();
+    }
+
+    @Override
+    protected String getMalformedValue() {
+        return RandomStrings.randomAsciiOfLength(random(), 8);
+    }
+}