Add parent filter query and BitsetProducer in NestedObjectMapper context (#109417)

server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java

@@ -9,7 +9,9 @@
 package org.elasticsearch.index.mapper;
 
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.join.BitSetProducer;
 import org.elasticsearch.common.Explicit;
+import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
@@ -18,6 +20,7 @@ import org.elasticsearch.xcontent.XContentBuilder;
 import java.io.IOException;
 import java.util.Locale;
 import java.util.Map;
+import java.util.function.Function;
 
 /**
  * A Mapper for nested objects
@@ -31,10 +34,12 @@ public class NestedObjectMapper extends ObjectMapper {
         private Explicit<Boolean> includeInRoot = Explicit.IMPLICIT_FALSE;
         private Explicit<Boolean> includeInParent = Explicit.IMPLICIT_FALSE;
         private final IndexVersion indexCreatedVersion;
+        private final Function<Query, BitSetProducer> bitsetProducer;
 
-        public Builder(String name, IndexVersion indexCreatedVersion) {
+        public Builder(String name, IndexVersion indexCreatedVersion, Function<Query, BitSetProducer> bitSetProducer) {
             super(name, Explicit.IMPLICIT_TRUE);
             this.indexCreatedVersion = indexCreatedVersion;
+            this.bitsetProducer = bitSetProducer;
         }
 
         Builder includeInRoot(boolean includeInRoot) {
@@ -50,24 +55,21 @@ public class NestedObjectMapper extends ObjectMapper {
         @Override
         public NestedObjectMapper build(MapperBuilderContext context) {
             boolean parentIncludedInRoot = this.includeInRoot.value();
+            final Query parentTypeFilter;
             if (context instanceof NestedMapperBuilderContext nc) {
                 // we're already inside a nested mapper, so adjust our includes
                 if (nc.parentIncludedInRoot && this.includeInParent.value()) {
                     this.includeInRoot = Explicit.IMPLICIT_FALSE;
                 }
+                parentTypeFilter = nc.nestedTypeFilter;
             } else {
                 // this is a top-level nested mapper, so include_in_parent = include_in_root
                 parentIncludedInRoot |= this.includeInParent.value();
                 if (this.includeInParent.value()) {
                     this.includeInRoot = Explicit.IMPLICIT_FALSE;
                 }
+                parentTypeFilter = Queries.newNonNestedFilter(indexCreatedVersion);
             }
-            NestedMapperBuilderContext nestedContext = new NestedMapperBuilderContext(
-                context.buildFullName(name()),
-                parentIncludedInRoot,
-                context.getDynamic(dynamic),
-                context.getMergeReason()
-            );
             final String fullPath = context.buildFullName(name());
             final String nestedTypePath;
             if (indexCreatedVersion.before(IndexVersions.V_8_0_0)) {
@@ -75,6 +77,14 @@ public class NestedObjectMapper extends ObjectMapper {
             } else {
                 nestedTypePath = fullPath;
             }
+            final Query nestedTypeFilter = NestedPathFieldMapper.filter(indexCreatedVersion, nestedTypePath);
+            NestedMapperBuilderContext nestedContext = new NestedMapperBuilderContext(
+                context.buildFullName(name()),
+                nestedTypeFilter,
+                parentIncludedInRoot,
+                context.getDynamic(dynamic),
+                context.getMergeReason()
+            );
             return new NestedObjectMapper(
                 name(),
                 fullPath,
@@ -83,8 +93,10 @@ public class NestedObjectMapper extends ObjectMapper {
                 dynamic,
                 includeInParent,
                 includeInRoot,
+                parentTypeFilter,
                 nestedTypePath,
-                NestedPathFieldMapper.filter(indexCreatedVersion, nestedTypePath)
+                nestedTypeFilter,
+                bitsetProducer
             );
         }
     }
@@ -96,7 +108,11 @@ public class NestedObjectMapper extends ObjectMapper {
             if (parseSubobjects(node).explicit()) {
                 throw new MapperParsingException("Nested type [" + name + "] does not support [subobjects] parameter");
             }
-            NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder(name, parserContext.indexVersionCreated());
+            NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder(
+                name,
+                parserContext.indexVersionCreated(),
+                parserContext::bitSetProducer
+            );
             parseNested(name, node, builder);
             parseObjectFields(node, parserContext, builder);
             return builder;
@@ -119,24 +135,43 @@ public class NestedObjectMapper extends ObjectMapper {
     }
 
     private static class NestedMapperBuilderContext extends MapperBuilderContext {
-
         final boolean parentIncludedInRoot;
-
-        NestedMapperBuilderContext(String path, boolean parentIncludedInRoot, Dynamic dynamic, MapperService.MergeReason mergeReason) {
+        final Query nestedTypeFilter;
+
+        NestedMapperBuilderContext(
+            String path,
+            Query nestedTypeFilter,
+            boolean parentIncludedInRoot,
+            Dynamic dynamic,
+            MapperService.MergeReason mergeReason
+        ) {
             super(path, false, false, false, dynamic, mergeReason);
             this.parentIncludedInRoot = parentIncludedInRoot;
+            this.nestedTypeFilter = nestedTypeFilter;
         }
 
         @Override
         public MapperBuilderContext createChildContext(String name, Dynamic dynamic) {
-            return new NestedMapperBuilderContext(buildFullName(name), parentIncludedInRoot, getDynamic(dynamic), getMergeReason());
+            return new NestedMapperBuilderContext(
+                buildFullName(name),
+                nestedTypeFilter,
+                parentIncludedInRoot,
+                getDynamic(dynamic),
+                getMergeReason()
+            );
         }
     }
 
     private final Explicit<Boolean> includeInRoot;
     private final Explicit<Boolean> includeInParent;
+    // The query to identify parent documents
+    private final Query parentTypeFilter;
+    // The path of the nested field
     private final String nestedTypePath;
+    // The query to identify nested documents at this level
     private final Query nestedTypeFilter;
+    // Function to create a bitset for identifying parent documents
+    private final Function<Query, BitSetProducer> bitsetProducer;
 
     NestedObjectMapper(
         String name,
@@ -146,14 +181,22 @@ public class NestedObjectMapper extends ObjectMapper {
         ObjectMapper.Dynamic dynamic,
         Explicit<Boolean> includeInParent,
         Explicit<Boolean> includeInRoot,
+        Query parentTypeFilter,
         String nestedTypePath,
-        Query nestedTypeFilter
+        Query nestedTypeFilter,
+        Function<Query, BitSetProducer> bitsetProducer
     ) {
         super(name, fullPath, enabled, Explicit.IMPLICIT_TRUE, Explicit.IMPLICIT_FALSE, dynamic, mappers);
+        this.parentTypeFilter = parentTypeFilter;
         this.nestedTypePath = nestedTypePath;
         this.nestedTypeFilter = nestedTypeFilter;
         this.includeInParent = includeInParent;
         this.includeInRoot = includeInRoot;
+        this.bitsetProducer = bitsetProducer;
+    }
+
+    public Query parentTypeFilter() {
+        return parentTypeFilter;
     }
 
     public Query nestedTypeFilter() {
@@ -177,13 +220,17 @@ public class NestedObjectMapper extends ObjectMapper {
         return this.includeInRoot.value();
     }
 
+    public Function<Query, BitSetProducer> bitsetProducer() {
+        return bitsetProducer;
+    }
+
     public Map<String, Mapper> getChildren() {
         return this.mappers;
     }
 
     @Override
     public ObjectMapper.Builder newBuilder(IndexVersion indexVersionCreated) {
-        NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder(simpleName(), indexVersionCreated);
+        NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder(simpleName(), indexVersionCreated, bitsetProducer);
         builder.enabled = enabled;
         builder.dynamic = dynamic;
         builder.includeInRoot = includeInRoot;
@@ -201,8 +248,10 @@ public class NestedObjectMapper extends ObjectMapper {
             dynamic,
             includeInParent,
             includeInRoot,
+            parentTypeFilter,
             nestedTypePath,
-            nestedTypeFilter
+            nestedTypeFilter,
+            bitsetProducer
         );
     }
 
@@ -270,8 +319,10 @@ public class NestedObjectMapper extends ObjectMapper {
             mergeResult.dynamic(),
             incInParent,
             incInRoot,
+            parentTypeFilter,
             nestedTypePath,
-            nestedTypeFilter
+            nestedTypeFilter,
+            bitsetProducer
         );
     }
 
@@ -285,6 +336,7 @@ public class NestedObjectMapper extends ObjectMapper {
         return mapperMergeContext.createChildContext(
             new NestedMapperBuilderContext(
                 mapperBuilderContext.buildFullName(name),
+                nestedTypeFilter,
                 parentIncludedInRoot,
                 mapperBuilderContext.getDynamic(dynamic),
                 mapperBuilderContext.getMergeReason()

server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java

@@ -176,7 +176,9 @@ public class FieldAliasMapperValidationTests extends ESTestCase {
     }
 
     private static NestedObjectMapper createNestedObjectMapper(String name) {
-        return new NestedObjectMapper.Builder(name, IndexVersion.current()).build(MapperBuilderContext.root(false, false));
+        return new NestedObjectMapper.Builder(name, IndexVersion.current(), query -> { throw new UnsupportedOperationException(); }).build(
+            MapperBuilderContext.root(false, false)
+        );
     }
 
     private static MappingLookup createMappingLookup(

server/src/test/java/org/elasticsearch/index/mapper/NestedLookupTests.java

@@ -64,7 +64,9 @@ public class NestedLookupTests extends MapperServiceTestCase {
     }
 
     private static NestedObjectMapper buildMapper(String name) {
-        return new NestedObjectMapper.Builder(name, IndexVersion.current()).build(MapperBuilderContext.root(false, false));
+        return new NestedObjectMapper.Builder(name, IndexVersion.current(), query -> { throw new UnsupportedOperationException(); }).build(
+            MapperBuilderContext.root(false, false)
+        );
     }
 
     public void testAllParentFilters() {

server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java

@@ -12,6 +12,7 @@ import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.index.IndexVersion;
@@ -1500,12 +1501,12 @@ public class NestedObjectMapperTests extends MapperServiceTestCase {
     }
 
     public void testMergeNested() {
-        NestedObjectMapper firstMapper = new NestedObjectMapper.Builder("nested1", IndexVersion.current()).includeInParent(true)
-            .includeInRoot(true)
-            .build(MapperBuilderContext.root(false, false));
-        NestedObjectMapper secondMapper = new NestedObjectMapper.Builder("nested1", IndexVersion.current()).includeInParent(false)
-            .includeInRoot(true)
-            .build(MapperBuilderContext.root(false, false));
+        NestedObjectMapper firstMapper = new NestedObjectMapper.Builder("nested1", IndexVersion.current(), query -> {
+            throw new UnsupportedOperationException();
+        }).includeInParent(true).includeInRoot(true).build(MapperBuilderContext.root(false, false));
+        NestedObjectMapper secondMapper = new NestedObjectMapper.Builder("nested1", IndexVersion.current(), query -> {
+            throw new UnsupportedOperationException();
+        }).includeInParent(false).includeInRoot(true).build(MapperBuilderContext.root(false, false));
 
         MapperException e = expectThrows(
             MapperException.class,
@@ -1533,6 +1534,39 @@ public class NestedObjectMapperTests extends MapperServiceTestCase {
         assertThat(object.withoutMappers().toString(), equalTo(shallowObject.toString()));
     }
 
+    public void testNestedMapperFilters() throws Exception {
+        DocumentMapper docMapper = createDocumentMapper(mapping(b -> {
+            b.startObject("nested1");
+            {
+                b.field("type", "nested");
+                b.startObject("properties");
+                {
+                    b.startObject("field1").field("type", "text").endObject();
+                    b.startObject("sub_nested");
+                    {
+                        b.field("type", "nested");
+                        b.startObject("properties");
+                        {
+                            b.startObject("field2").field("type", "text").endObject();
+                        }
+                        b.endObject();
+                    }
+                    b.endObject();
+                }
+                b.endObject();
+            }
+            b.endObject();
+        }));
+
+        assertThat(docMapper.mappers().nestedLookup().getNestedMappers().size(), equalTo(2));
+        assertThat(docMapper.mappers().nestedLookup().getNestedMappers().get("nested1"), instanceOf(NestedObjectMapper.class));
+        NestedObjectMapper mapper1 = docMapper.mappers().nestedLookup().getNestedMappers().get("nested1");
+        assertThat(mapper1.parentTypeFilter(), equalTo(Queries.newNonNestedFilter(IndexVersion.current())));
+
+        NestedObjectMapper mapper2 = docMapper.mappers().nestedLookup().getNestedMappers().get("nested1.sub_nested");
+        assertThat(mapper2.parentTypeFilter(), equalTo(mapper1.nestedTypeFilter()));
+    }
+
     private NestedObjectMapper createNestedObjectMapperWithAllParametersSet(CheckedConsumer<XContentBuilder, IOException> propertiesBuilder)
         throws IOException {
         DocumentMapper mapper = createDocumentMapper(mapping(b -> {

server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java

@@ -912,6 +912,8 @@ public class NestedAggregatorTests extends AggregatorTestCase {
     );
 
     public static NestedObjectMapper nestedObject(String path) {
-        return new NestedObjectMapper.Builder(path, IndexVersion.current()).build(MapperBuilderContext.root(false, false));
+        return new NestedObjectMapper.Builder(path, IndexVersion.current(), query -> { throw new UnsupportedOperationException(); }).build(
+            MapperBuilderContext.root(false, false)
+        );
     }
 }

server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java

@@ -194,9 +194,9 @@ public abstract class AbstractSortTestCase<T extends SortBuilder<T>> extends EST
             IndexFieldData.Builder builder = fieldType.fielddataBuilder(fdc);
             return builder.build(new IndexFieldDataCache.None(), null);
         };
-        NestedLookup nestedLookup = NestedLookup.build(
-            List.of(new NestedObjectMapper.Builder("path", IndexVersion.current()).build(MapperBuilderContext.root(false, false)))
-        );
+        NestedLookup nestedLookup = NestedLookup.build(List.of(new NestedObjectMapper.Builder("path", IndexVersion.current(), query -> {
+            throw new UnsupportedOperationException();
+        }).build(MapperBuilderContext.root(false, false))));
         return new SearchExecutionContext(
             0,
             0,

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java

@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.inference.mapper;
 
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.join.BitSetProducer;
 import org.apache.lucene.search.join.ScoreMode;
 import org.elasticsearch.cluster.metadata.InferenceFieldMetadata;
 import org.elasticsearch.common.Explicit;
@@ -78,7 +79,7 @@ public class SemanticTextFieldMapper extends FieldMapper implements InferenceFie
     public static final String CONTENT_TYPE = "semantic_text";
 
     public static final TypeParser PARSER = new TypeParser(
-        (n, c) -> new Builder(n, c.indexVersionCreated()),
+        (n, c) -> new Builder(n, c.indexVersionCreated(), c::bitSetProducer),
         List.of(notInMultiFields(CONTENT_TYPE), notFromDynamicTemplates(CONTENT_TYPE))
     );
 
@@ -110,10 +111,10 @@ public class SemanticTextFieldMapper extends FieldMapper implements InferenceFie
 
         private Function<MapperBuilderContext, ObjectMapper> inferenceFieldBuilder;
 
-        public Builder(String name, IndexVersion indexVersionCreated) {
+        public Builder(String name, IndexVersion indexVersionCreated, Function<Query, BitSetProducer> bitSetProducer) {
             super(name);
             this.indexVersionCreated = indexVersionCreated;
-            this.inferenceFieldBuilder = c -> createInferenceField(c, indexVersionCreated, modelSettings.get());
+            this.inferenceFieldBuilder = c -> createInferenceField(c, indexVersionCreated, modelSettings.get(), bitSetProducer);
         }
 
         public Builder setInferenceId(String id) {
@@ -181,7 +182,7 @@ public class SemanticTextFieldMapper extends FieldMapper implements InferenceFie
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new Builder(simpleName(), fieldType().indexVersionCreated).init(this);
+        return new Builder(simpleName(), fieldType().indexVersionCreated, fieldType().getChunksField().bitsetProducer()).init(this);
     }
 
     @Override
@@ -219,7 +220,11 @@ public class SemanticTextFieldMapper extends FieldMapper implements InferenceFie
         final SemanticTextFieldMapper mapper;
         if (fieldType().getModelSettings() == null) {
             context.path().remove();
-            Builder builder = (Builder) new Builder(simpleName(), fieldType().indexVersionCreated).init(this);
+            Builder builder = (Builder) new Builder(
+                simpleName(),
+                fieldType().indexVersionCreated,
+                fieldType().getChunksField().bitsetProducer()
+            ).init(this);
             try {
                 mapper = builder.setModelSettings(field.inference().modelSettings())
                     .setInferenceId(field.inference().inferenceId())
@@ -441,18 +446,20 @@ public class SemanticTextFieldMapper extends FieldMapper implements InferenceFie
     private static ObjectMapper createInferenceField(
         MapperBuilderContext context,
         IndexVersion indexVersionCreated,
-        @Nullable SemanticTextField.ModelSettings modelSettings
+        @Nullable SemanticTextField.ModelSettings modelSettings,
+        Function<Query, BitSetProducer> bitSetProducer
     ) {
         return new ObjectMapper.Builder(INFERENCE_FIELD, Explicit.EXPLICIT_TRUE).dynamic(ObjectMapper.Dynamic.FALSE)
-            .add(createChunksField(indexVersionCreated, modelSettings))
+            .add(createChunksField(indexVersionCreated, modelSettings, bitSetProducer))
             .build(context);
     }
 
     private static NestedObjectMapper.Builder createChunksField(
         IndexVersion indexVersionCreated,
-        SemanticTextField.ModelSettings modelSettings
+        @Nullable SemanticTextField.ModelSettings modelSettings,
+        Function<Query, BitSetProducer> bitSetProducer
     ) {
-        NestedObjectMapper.Builder chunksField = new NestedObjectMapper.Builder(CHUNKS_FIELD, indexVersionCreated);
+        NestedObjectMapper.Builder chunksField = new NestedObjectMapper.Builder(CHUNKS_FIELD, indexVersionCreated, bitSetProducer);
         chunksField.dynamic(ObjectMapper.Dynamic.FALSE);
         KeywordFieldMapper.Builder chunkTextField = new KeywordFieldMapper.Builder(CHUNKED_TEXT_FIELD, indexVersionCreated).indexed(false)
             .docValues(false);