Store arrays offsets for boolean fields natively with synthetic source (#125529) (#125596)

This patch builds on the work in #113757, #122999, and #124594 to natively
store array offsets for boolean fields instead of falling back to ignored
source when `synthetic_source_keep: arrays`.

(cherry picked from commit af1f1452d78ac180a6f6c90822ebd487266babbe)

# Conflicts:
#	server/src/main/java/org/elasticsearch/index/IndexVersions.java
#	server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java

docs/changelog/125529.yaml

@@ -0,0 +1,5 @@
+pr: 125529
+summary: Store arrays offsets for boolean fields natively with synthetic source
+area: Mapping
+type: enhancement
+issues: []

server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java

@@ -28,6 +28,7 @@ import org.elasticsearch.core.Booleans;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.fielddata.FieldDataContext;
 import org.elasticsearch.index.fielddata.IndexFieldData;
@@ -48,6 +49,7 @@ import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.time.ZoneId;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
@@ -56,6 +58,8 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 
+import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName;
+
 /**
  * A field mapper for boolean fields.
  */
@@ -102,9 +106,17 @@ public class BooleanFieldMapper extends FieldMapper {
 
         private final IndexVersion indexCreatedVersion;
 
+        private final SourceKeepMode indexSourceKeepMode;
+
         private final Parameter<Boolean> dimension;
 
-        public Builder(String name, ScriptCompiler scriptCompiler, boolean ignoreMalformedByDefault, IndexVersion indexCreatedVersion) {
+        public Builder(
+            String name,
+            ScriptCompiler scriptCompiler,
+            boolean ignoreMalformedByDefault,
+            IndexVersion indexCreatedVersion,
+            SourceKeepMode indexSourceKeepMode
+        ) {
             super(name);
             this.scriptCompiler = Objects.requireNonNull(scriptCompiler);
             this.indexCreatedVersion = Objects.requireNonNull(indexCreatedVersion);
@@ -129,6 +141,8 @@ public class BooleanFieldMapper extends FieldMapper {
                     );
                 }
             });
+
+            this.indexSourceKeepMode = indexSourceKeepMode;
         }
 
         public Builder dimension(boolean dimension) {
@@ -168,7 +182,23 @@ public class BooleanFieldMapper extends FieldMapper {
             );
             hasScript = script.get() != null;
             onScriptError = onScriptErrorParam.getValue();
-            return new BooleanFieldMapper(leafName(), ft, builderParams(this, context), context.isSourceSynthetic(), this);
+            String offsetsFieldName = getOffsetsFieldName(
+                context,
+                indexSourceKeepMode,
+                docValues.getValue(),
+                stored.getValue(),
+                this,
+                indexCreatedVersion,
+                IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY
+            );
+            return new BooleanFieldMapper(
+                leafName(),
+                ft,
+                builderParams(this, context),
+                context.isSourceSynthetic(),
+                this,
+                offsetsFieldName
+            );
         }
 
         private FieldValues<Boolean> scriptValues() {
@@ -187,7 +217,13 @@ public class BooleanFieldMapper extends FieldMapper {
     private static final IndexVersion MINIMUM_COMPATIBILITY_VERSION = IndexVersion.fromId(5000099);
 
     public static final TypeParser PARSER = new TypeParser(
-        (n, c) -> new Builder(n, c.scriptCompiler(), IGNORE_MALFORMED_SETTING.get(c.getSettings()), c.indexVersionCreated()),
+        (n, c) -> new Builder(
+            n,
+            c.scriptCompiler(),
+            IGNORE_MALFORMED_SETTING.get(c.getSettings()),
+            c.indexVersionCreated(),
+            c.getIndexSettings().sourceKeepMode()
+        ),
         MINIMUM_COMPATIBILITY_VERSION
     );
 
@@ -490,12 +526,16 @@ public class BooleanFieldMapper extends FieldMapper {
 
     private final boolean storeMalformedFields;
 
+    private final String offsetsFieldName;
+    private final SourceKeepMode indexSourceKeepMode;
+
     protected BooleanFieldMapper(
         String simpleName,
         MappedFieldType mappedFieldType,
         BuilderParams builderParams,
         boolean storeMalformedFields,
-        Builder builder
+        Builder builder,
+        String offsetsFieldName
     ) {
         super(simpleName, mappedFieldType, builderParams);
         this.nullValue = builder.nullValue.getValue();
@@ -509,6 +549,8 @@ public class BooleanFieldMapper extends FieldMapper {
         this.ignoreMalformed = builder.ignoreMalformed.getValue();
         this.ignoreMalformedByDefault = builder.ignoreMalformed.getDefaultValue().value();
         this.storeMalformedFields = storeMalformedFields;
+        this.offsetsFieldName = offsetsFieldName;
+        this.indexSourceKeepMode = builder.indexSourceKeepMode;
     }
 
     @Override
@@ -521,6 +563,11 @@ public class BooleanFieldMapper extends FieldMapper {
         return (BooleanFieldType) super.fieldType();
     }
 
+    @Override
+    public String getOffsetFieldName() {
+        return offsetsFieldName;
+    }
+
     @Override
     protected void parseCreateField(DocumentParserContext context) throws IOException {
         if (indexed == false && stored == false && hasDocValues == false) {
@@ -543,12 +590,20 @@ public class BooleanFieldMapper extends FieldMapper {
                         // Save a copy of the field so synthetic source can load it
                         context.doc().add(IgnoreMalformedStoredValues.storedField(fullPath(), context.parser()));
                     }
+                    return;
                 } else {
                     throw e;
                 }
             }
         }
         indexValue(context, value);
+        if (offsetsFieldName != null && context.isImmediateParentAnArray() && context.canAddIgnoredField()) {
+            if (value != null) {
+                context.getOffSetContext().recordOffset(offsetsFieldName, value);
+            } else {
+                context.getOffSetContext().recordNull(offsetsFieldName);
+            }
+        }
     }
 
     private void indexValue(DocumentParserContext context, Boolean value) {
@@ -584,8 +639,9 @@ public class BooleanFieldMapper extends FieldMapper {
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new Builder(leafName(), scriptCompiler, ignoreMalformedByDefault, indexCreatedVersion).dimension(fieldType().isDimension())
-            .init(this);
+        return new Builder(leafName(), scriptCompiler, ignoreMalformedByDefault, indexCreatedVersion, indexSourceKeepMode).dimension(
+            fieldType().isDimension()
+        ).init(this);
     }
 
     @Override
@@ -607,17 +663,34 @@ public class BooleanFieldMapper extends FieldMapper {
         return CONTENT_TYPE;
     }
 
+    private SourceLoader.SyntheticFieldLoader docValuesSyntheticFieldLoader() {
+        if (offsetsFieldName != null) {
+            var layers = new ArrayList<CompositeSyntheticFieldLoader.Layer>();
+            layers.add(
+                new SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer(
+                    fullPath(),
+                    offsetsFieldName,
+                    (b, value) -> b.value(value == 1)
+                )
+            );
+            if (ignoreMalformed.value()) {
+                layers.add(new CompositeSyntheticFieldLoader.MalformedValuesLayer(fullPath()));
+            }
+            return new CompositeSyntheticFieldLoader(leafName(), fullPath(), layers);
+        } else {
+            return new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed.value()) {
+                @Override
+                protected void writeValue(XContentBuilder b, long value) throws IOException {
+                    b.value(value == 1);
+                }
+            };
+        }
+    }
+
     @Override
     protected SyntheticSourceSupport syntheticSourceSupport() {
         if (hasDocValues) {
-            return new SyntheticSourceSupport.Native(
-                () -> new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed.value()) {
-                    @Override
-                    protected void writeValue(XContentBuilder b, long value) throws IOException {
-                        b.value(value == 1);
-                    }
-                }
-            );
+            return new SyntheticSourceSupport.Native(this::docValuesSyntheticFieldLoader);
         }
 
         return super.syntheticSourceSupport();

server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java

@@ -387,7 +387,8 @@ final class DynamicFieldsBuilder {
                     name,
                     ScriptCompiler.NONE,
                     ignoreMalformed,
-                    context.indexSettings().getIndexVersionCreated()
+                    context.indexSettings().getIndexVersionCreated(),
+                    context.indexSettings().sourceKeepMode()
                 ),
                 context
             );

server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java

@@ -312,43 +312,68 @@ public class BooleanFieldMapperTests extends MapperTestCase {
         return true;
     }
 
-    @Override
-    protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
-        return new SyntheticSourceSupport() {
-            Boolean nullValue = usually() ? null : randomBoolean();
+    private class BooleanSyntheticSourceSupport implements SyntheticSourceSupport {
+        Boolean nullValue = usually() ? null : randomBoolean();
+        private boolean ignoreMalformed;
 
-            @Override
-            public SyntheticSourceExample example(int maxVals) throws IOException {
-                if (randomBoolean()) {
-                    Tuple<Boolean, Boolean> v = generateValue();
-                    return new SyntheticSourceExample(v.v1(), v.v2(), this::mapping);
-                }
-                List<Tuple<Boolean, Boolean>> values = randomList(1, maxVals, this::generateValue);
-                List<Boolean> in = values.stream().map(Tuple::v1).toList();
-                List<Boolean> outList = values.stream().map(Tuple::v2).sorted().toList();
-                Object out = outList.size() == 1 ? outList.get(0) : outList;
-                return new SyntheticSourceExample(in, out, this::mapping);
+        BooleanSyntheticSourceSupport(boolean ignoreMalformed) {
+            this.ignoreMalformed = ignoreMalformed;
+        }
+
+        @Override
+        public SyntheticSourceExample example(int maxVals) throws IOException {
+            if (randomBoolean()) {
+                Tuple<Boolean, Boolean> v = generateValue();
+                return new SyntheticSourceExample(v.v1(), v.v2(), this::mapping);
             }
+            List<Tuple<Boolean, Boolean>> values = randomList(1, maxVals, this::generateValue);
+            List<Boolean> in = values.stream().map(Tuple::v1).toList();
+            List<Boolean> outList = values.stream().map(Tuple::v2).sorted().toList();
+            Object out = outList.size() == 1 ? outList.get(0) : outList;
+            return new SyntheticSourceExample(in, out, this::mapping);
+        }
 
-            private Tuple<Boolean, Boolean> generateValue() {
-                if (nullValue != null && randomBoolean()) {
-                    return Tuple.tuple(null, nullValue);
-                }
-                boolean b = randomBoolean();
-                return Tuple.tuple(b, b);
+        private Tuple<Boolean, Boolean> generateValue() {
+            if (nullValue != null && randomBoolean()) {
+                return Tuple.tuple(null, nullValue);
             }
+            boolean b = randomBoolean();
+            return Tuple.tuple(b, b);
+        }
 
-            private void mapping(XContentBuilder b) throws IOException {
-                minimalMapping(b);
-                if (nullValue != null) {
-                    b.field("null_value", nullValue);
-                }
-                b.field("ignore_malformed", ignoreMalformed);
+        private void mapping(XContentBuilder b) throws IOException {
+            minimalMapping(b);
+            if (nullValue != null) {
+                b.field("null_value", nullValue);
             }
+            b.field("ignore_malformed", ignoreMalformed);
+        }
+
+        @Override
+        public List<SyntheticSourceInvalidExample> invalidExample() throws IOException {
+            return List.of();
+        }
+    };
 
+    @Override
+    protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
+        return new BooleanSyntheticSourceSupport(ignoreMalformed);
+    }
+
+    @Override
+    protected SyntheticSourceSupport syntheticSourceSupportForKeepTests(boolean ignoreMalformed, Mapper.SourceKeepMode keepMode) {
+        return new BooleanSyntheticSourceSupport(ignoreMalformed) {
             @Override
-            public List<SyntheticSourceInvalidExample> invalidExample() throws IOException {
-                return List.of();
+            public SyntheticSourceExample example(int maxVals) throws IOException {
+                var example = super.example(maxVals);
+                // Need the expectedForSyntheticSource as inputValue since MapperTestCase#testSyntheticSourceKeepArrays
+                // uses the inputValue as both the input and expected.
+                return new SyntheticSourceExample(
+                    example.expectedForSyntheticSource(),
+                    example.expectedForSyntheticSource(),
+                    example.expectedForBlockLoader(),
+                    example.mapping()
+                );
             }
         };
     }

server/src/test/java/org/elasticsearch/index/mapper/BooleanOffsetDocValuesLoaderTests.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.mapper;
+
+public class BooleanOffsetDocValuesLoaderTests extends OffsetDocValuesLoaderTestCase {
+
+    public void testOffsetArray() throws Exception {
+        verifyOffsets("{\"field\":[true,false,true,true,false,true]}");
+        verifyOffsets("{\"field\":[true,null,false,false,null,null,true,false]}");
+        verifyOffsets("{\"field\":[true,true,true,true]}");
+    }
+
+    public void testOffsetNestedArray() throws Exception {
+        verifyOffsets("{\"field\":[[\"true\",[false,[true]]],[\"true\",false,true]]}", "{\"field\":[true,false,true,true,false,true]}");
+        verifyOffsets(
+            "{\"field\":[true,[null,[[false,false],[null,null]],[true,false]]]}",
+            "{\"field\":[true,null,false,false,null,null,true,false]}"
+        );
+    }
+
+    @Override
+    protected String getFieldTypeName() {
+        return "boolean";
+    }
+
+    @Override
+    protected Object randomValue() {
+        return randomBoolean();
+    }
+}

server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java

@@ -354,7 +354,7 @@ public class BooleanScriptFieldTypeTests extends AbstractNonTextScriptFieldTypeT
     }
 
     public void testDualingQueries() throws IOException {
-        BooleanFieldMapper ootb = new BooleanFieldMapper.Builder("foo", ScriptCompiler.NONE, false, IndexVersion.current()).build(
+        BooleanFieldMapper ootb = new BooleanFieldMapper.Builder("foo", ScriptCompiler.NONE, false, IndexVersion.current(), null).build(
             MapperBuilderContext.root(false, false)
         );
         try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) {

server/src/test/java/org/elasticsearch/index/mapper/BooleanSyntheticSourceNativeArrayIntegrationTests.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.mapper;
+
+import com.carrotsearch.randomizedtesting.generators.RandomStrings;
+
+public class BooleanSyntheticSourceNativeArrayIntegrationTests extends NativeArrayIntegrationTestCase {
+    @Override
+    protected String getFieldTypeName() {
+        return "boolean";
+    }
+
+    @Override
+    protected Object getRandomValue() {
+        return randomBoolean();
+    }
+
+    @Override
+    protected Object getMalformedValue() {
+        return RandomStrings.randomAsciiOfLength(random(), 8);
+    }
+}

server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java

@@ -160,7 +160,7 @@ public class FieldAliasMapperValidationTests extends ESTestCase {
     }
 
     private static FieldMapper createFieldMapper(String parent, String name) {
-        return new BooleanFieldMapper.Builder(name, ScriptCompiler.NONE, false, IndexVersion.current()).build(
+        return new BooleanFieldMapper.Builder(name, ScriptCompiler.NONE, false, IndexVersion.current(), null).build(
             new MapperBuilderContext(
                 parent,
                 false,

server/src/test/java/org/elasticsearch/index/mapper/MultiFieldsSerializationTests.java

@@ -38,10 +38,10 @@ public class MultiFieldsSerializationTests extends ESTestCase {
         sortedNames.sort(Comparator.naturalOrder());
 
         for (String name : names) {
-            builder.add(new BooleanFieldMapper.Builder(name, ScriptCompiler.NONE, false, IndexVersion.current()));
+            builder.add(new BooleanFieldMapper.Builder(name, ScriptCompiler.NONE, false, IndexVersion.current(), null));
         }
 
-        Mapper.Builder root = new BooleanFieldMapper.Builder("root", ScriptCompiler.NONE, false, IndexVersion.current());
+        Mapper.Builder root = new BooleanFieldMapper.Builder("root", ScriptCompiler.NONE, false, IndexVersion.current(), null);
         FieldMapper.MultiFields multiFields = builder.build(root, MapperBuilderContext.root(false, false));
 
         String serialized = Strings.toString(multiFields);