Return docs when using nested mappings in archive indices (#90585)

Archive indices < 6.1 do not work together with nested fields. While the documentation makes sure not to claim support for nested fields, see https://www.elastic.co/guide/en/elasticsearch/reference/current/archive-indices.html#archive-indices-supported-field-types, it leads to a situation where the import still works yet none of the documents at all are returned by any query (not even match_all). This is because indices before 6.1 did not have primary terms, but the default search context in ES 8 adds a FieldExistsQuery(SeqNoFieldMapper.PRIMARY_TERM_NAME) filter to the query:

https://github.com/elastic/elasticsearch/blob/f56126089ca4db89b631901ad7cce0a8e10e2fe5/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java#L284

This PR fixes the issue by adding basic support for nested fields so that it at least allows queries that are not leveraging
the nested documents to work (i.e. allow extracting the documents e.g. to be reindexed).

Closes #90523

docs/changelog/90585.yaml

@@ -0,0 +1,6 @@
+pr: 90585
+summary: Return docs when using nested mappings in archive indices
+area: Search
+type: enhancement
+issues:
+ - 90523

modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java

@@ -276,7 +276,7 @@ public class PercolatorFieldMapper extends FieldMapper {
             }
             Query filter = null;
             if (excludeNestedDocuments) {
-                filter = Queries.newNonNestedFilter();
+                filter = Queries.newNonNestedFilter(indexVersion);
             }
             return new PercolateQuery(name, queryStore, documents, candidateQuery, searcher, filter, verifiedMatchesQuery);
         }

modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhase.java

@@ -20,6 +20,7 @@ import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.util.BitSet;
 import org.apache.lucene.util.BitSetIterator;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.document.DocumentField;
 import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.search.fetch.FetchContext;
@@ -51,7 +52,9 @@ final class PercolatorMatchedSlotSubFetchPhase implements FetchSubPhase {
         List<PercolateQuery> percolateQueries = locatePercolatorQuery(fetchContext.query());
         boolean singlePercolateQuery = percolateQueries.size() == 1;
         for (PercolateQuery pq : percolateQueries) {
-            percolateContexts.add(new PercolateContext(pq, singlePercolateQuery));
+            percolateContexts.add(
+                new PercolateContext(pq, singlePercolateQuery, fetchContext.getSearchExecutionContext().indexVersionCreated())
+            );
         }
         if (percolateContexts.isEmpty()) {
             return null;
@@ -75,7 +78,7 @@ final class PercolatorMatchedSlotSubFetchPhase implements FetchSubPhase {
                         // This is not a document with a percolator field.
                         continue;
                     }
-                    query = pc.filterNestedDocs(query);
+                    query = pc.filterNestedDocs(query, fetchContext.getSearchExecutionContext().indexVersionCreated());
                     IndexSearcher percolatorIndexSearcher = pc.percolateQuery.getPercolatorIndexSearcher();
                     int memoryIndexMaxDoc = percolatorIndexSearcher.getIndexReader().maxDoc();
                     TopDocs topDocs = percolatorIndexSearcher.search(query, memoryIndexMaxDoc, new Sort(SortField.FIELD_DOC));
@@ -98,11 +101,11 @@ final class PercolatorMatchedSlotSubFetchPhase implements FetchSubPhase {
         final boolean singlePercolateQuery;
         final int[] rootDocsBySlot;
 
-        PercolateContext(PercolateQuery pq, boolean singlePercolateQuery) throws IOException {
+        PercolateContext(PercolateQuery pq, boolean singlePercolateQuery, Version indexVersionCreated) throws IOException {
             this.percolateQuery = pq;
             this.singlePercolateQuery = singlePercolateQuery;
             IndexSearcher percolatorIndexSearcher = percolateQuery.getPercolatorIndexSearcher();
-            Query nonNestedFilter = percolatorIndexSearcher.rewrite(Queries.newNonNestedFilter());
+            Query nonNestedFilter = percolatorIndexSearcher.rewrite(Queries.newNonNestedFilter(indexVersionCreated));
             Weight weight = percolatorIndexSearcher.createWeight(nonNestedFilter, ScoreMode.COMPLETE_NO_SCORES, 1f);
             Scorer s = weight.scorer(percolatorIndexSearcher.getIndexReader().leaves().get(0));
             int memoryIndexMaxDoc = percolatorIndexSearcher.getIndexReader().maxDoc();
@@ -119,11 +122,11 @@ final class PercolatorMatchedSlotSubFetchPhase implements FetchSubPhase {
             return singlePercolateQuery ? FIELD_NAME_PREFIX : FIELD_NAME_PREFIX + "_" + percolateQuery.getName();
         }
 
-        Query filterNestedDocs(Query in) {
+        Query filterNestedDocs(Query in, Version indexVersionCreated) {
             if (rootDocsBySlot != null) {
                 // Ensures that we filter out nested documents
                 return new BooleanQuery.Builder().add(in, BooleanClause.Occur.MUST)
-                    .add(Queries.newNonNestedFilter(), BooleanClause.Occur.FILTER)
+                    .add(Queries.newNonNestedFilter(indexVersionCreated), BooleanClause.Occur.FILTER)
                     .build();
             }
             return in;

modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhaseTests.java

@@ -23,7 +23,9 @@ import org.apache.lucene.search.TotalHits;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.tests.index.RandomIndexWriter;
 import org.apache.lucene.util.FixedBitSet;
+import org.elasticsearch.Version;
 import org.elasticsearch.index.mapper.SeqNoFieldMapper;
+import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.fetch.FetchContext;
 import org.elasticsearch.search.fetch.FetchSubPhase.HitContext;
@@ -70,6 +72,9 @@ public class PercolatorMatchedSlotSubFetchPhaseTests extends ESTestCase {
 
                     FetchContext sc = mock(FetchContext.class);
                     when(sc.query()).thenReturn(percolateQuery);
+                    SearchExecutionContext sec = mock(SearchExecutionContext.class);
+                    when(sc.getSearchExecutionContext()).thenReturn(sec);
+                    when(sec.indexVersionCreated()).thenReturn(Version.CURRENT);
 
                     FetchSubPhaseProcessor processor = phase.getProcessor(sc);
                     assertNotNull(processor);
@@ -98,6 +103,9 @@ public class PercolatorMatchedSlotSubFetchPhaseTests extends ESTestCase {
 
                     FetchContext sc = mock(FetchContext.class);
                     when(sc.query()).thenReturn(percolateQuery);
+                    SearchExecutionContext sec = mock(SearchExecutionContext.class);
+                    when(sc.getSearchExecutionContext()).thenReturn(sec);
+                    when(sec.indexVersionCreated()).thenReturn(Version.CURRENT);
 
                     FetchSubPhaseProcessor processor = phase.getProcessor(sc);
                     assertNotNull(processor);
@@ -125,6 +133,9 @@ public class PercolatorMatchedSlotSubFetchPhaseTests extends ESTestCase {
 
                     FetchContext sc = mock(FetchContext.class);
                     when(sc.query()).thenReturn(percolateQuery);
+                    SearchExecutionContext sec = mock(SearchExecutionContext.class);
+                    when(sc.getSearchExecutionContext()).thenReturn(sec);
+                    when(sec.indexVersionCreated()).thenReturn(Version.CURRENT);
 
                     FetchSubPhaseProcessor processor = phase.getProcessor(sc);
                     assertNotNull(processor);

server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java

@@ -1842,7 +1842,7 @@ public class IndexRecoveryIT extends AbstractIndexRecoveryIntegTestCase {
             final Query query = new BooleanQuery.Builder().add(
                 LongPoint.newRangeQuery(SeqNoFieldMapper.NAME, commitLocalCheckpoint + 1, Long.MAX_VALUE),
                 BooleanClause.Occur.MUST
-            ).add(Queries.newNonNestedFilter(), BooleanClause.Occur.MUST).build();
+            ).add(Queries.newNonNestedFilter(Version.CURRENT), BooleanClause.Occur.MUST).build();
             final Weight weight = searcher.createWeight(searcher.rewrite(query), ScoreMode.COMPLETE_NO_SCORES, 1.0f);
             for (LeafReaderContext leaf : directoryReader.leaves()) {
                 final Scorer scorer = weight.scorer(leaf);

server/src/main/java/org/elasticsearch/common/lucene/search/Queries.java

@@ -8,15 +8,20 @@
 
 package org.elasticsearch.common.lucene.search;
 
+import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.FieldExistsQuery;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.index.mapper.NestedPathFieldMapper;
 import org.elasticsearch.index.mapper.SeqNoFieldMapper;
 
 import java.util.Collection;
@@ -48,15 +53,30 @@ public class Queries {
         return Queries.newMatchNoDocsQuery("failed [" + field + "] query, caused by " + message);
     }
 
-    public static Query newNestedFilter() {
-        return not(newNonNestedFilter());
+    private static final Version NESTED_DOCS_IDENTIFIED_VIA_PRIMARY_TERMS_VERSION = Version.fromString("6.1.0");
+
+    /**
+     * Creates a new nested docs query
+     * @param indexVersionCreated the index version created since newer indices can identify a parent field more efficiently
+     */
+    public static Query newNestedFilter(Version indexVersionCreated) {
+        if (indexVersionCreated.onOrAfter(NESTED_DOCS_IDENTIFIED_VIA_PRIMARY_TERMS_VERSION)) {
+            return not(newNonNestedFilter(indexVersionCreated));
+        } else {
+            return new PrefixQuery(new Term(NestedPathFieldMapper.NAME_PRE_V8, new BytesRef("__")));
+        }
     }
 
     /**
      * Creates a new non-nested docs query
+     * @param indexVersionCreated the index version created since newer indices can identify a parent field more efficiently
      */
-    public static Query newNonNestedFilter() {
-        return new FieldExistsQuery(SeqNoFieldMapper.PRIMARY_TERM_NAME);
+    public static Query newNonNestedFilter(Version indexVersionCreated) {
+        if (indexVersionCreated.onOrAfter(NESTED_DOCS_IDENTIFIED_VIA_PRIMARY_TERMS_VERSION)) {
+            return new FieldExistsQuery(SeqNoFieldMapper.PRIMARY_TERM_NAME);
+        } else {
+            return not(newNestedFilter(indexVersionCreated));
+        }
     }
 
     public static BooleanQuery filtered(@Nullable Query query, @Nullable Query filter) {

server/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java

@@ -267,7 +267,7 @@ public final class BitsetFilterCache
             MappingLookup lookup = mapperService.mappingLookup();
             NestedLookup nestedLookup = lookup.nestedLookup();
             if (nestedLookup != NestedLookup.EMPTY) {
-                warmUp.add(Queries.newNonNestedFilter());
+                warmUp.add(Queries.newNonNestedFilter(mapperService.getIndexSettings().getIndexVersionCreated()));
                 warmUp.addAll(nestedLookup.getNestedParentFilters().values());
             }
 

server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java

@@ -265,7 +265,10 @@ public class InternalEngine extends Engine {
             maxSeqNoOfUpdatesOrDeletes = new AtomicLong(SequenceNumbers.max(localCheckpointTracker.getMaxSeqNo(), translog.getMaxSeqNo()));
             if (localCheckpointTracker.getPersistedCheckpoint() < localCheckpointTracker.getMaxSeqNo()) {
                 try (Searcher searcher = acquireSearcher("restore_version_map_and_checkpoint_tracker", SearcherScope.INTERNAL)) {
-                    restoreVersionMapAndCheckpointTracker(Lucene.wrapAllDocsLive(searcher.getDirectoryReader()));
+                    restoreVersionMapAndCheckpointTracker(
+                        Lucene.wrapAllDocsLive(searcher.getDirectoryReader()),
+                        engineConfig.getIndexSettings().getIndexVersionCreated()
+                    );
                 } catch (IOException e) {
                     throw new EngineCreationFailureException(
                         config().getShardId(),
@@ -2769,7 +2772,12 @@ public class InternalEngine extends Engine {
         ensureOpen();
         refreshIfNeeded(source, toSeqNo);
         try (Searcher searcher = acquireSearcher(source, SearcherScope.INTERNAL)) {
-            return LuceneChangesSnapshot.countOperations(searcher, fromSeqNo, toSeqNo);
+            return LuceneChangesSnapshot.countOperations(
+                searcher,
+                fromSeqNo,
+                toSeqNo,
+                config().getIndexSettings().getIndexVersionCreated()
+            );
         } catch (Exception e) {
             try {
                 maybeFailEngine("count changes", e);
@@ -2800,7 +2808,8 @@ public class InternalEngine extends Engine {
                 toSeqNo,
                 requiredFullRange,
                 singleConsumer,
-                accessStats
+                accessStats,
+                config().getIndexSettings().getIndexVersionCreated()
             );
             searcher = null;
             return snapshot;
@@ -2967,14 +2976,14 @@ public class InternalEngine extends Engine {
      * after the local checkpoint in the safe commit. This step ensures the live version map and checkpoint tracker
      * are in sync with the Lucene commit.
      */
-    private void restoreVersionMapAndCheckpointTracker(DirectoryReader directoryReader) throws IOException {
+    private void restoreVersionMapAndCheckpointTracker(DirectoryReader directoryReader, Version indexVersionCreated) throws IOException {
         final IndexSearcher searcher = new IndexSearcher(directoryReader);
         searcher.setQueryCache(null);
         final Query query = new BooleanQuery.Builder().add(
             LongPoint.newRangeQuery(SeqNoFieldMapper.NAME, getPersistedLocalCheckpoint() + 1, Long.MAX_VALUE),
             BooleanClause.Occur.MUST
         )
-            .add(Queries.newNonNestedFilter(), BooleanClause.Occur.MUST) // exclude non-root nested documents
+            .add(Queries.newNonNestedFilter(indexVersionCreated), BooleanClause.Occur.MUST) // exclude non-root nested documents
             .build();
         final Weight weight = searcher.createWeight(searcher.rewrite(query), ScoreMode.COMPLETE_NO_SCORES, 1.0f);
         for (LeafReaderContext leaf : directoryReader.leaves()) {

server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java

@@ -24,6 +24,7 @@ import org.apache.lucene.search.SortField;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.TopFieldCollector;
 import org.apache.lucene.util.ArrayUtil;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.index.SequentialStoredFieldsLeafReader;
@@ -62,6 +63,8 @@ final class LuceneChangesSnapshot implements Translog.Snapshot {
     private final ParallelArray parallelArray;
     private final Closeable onClose;
 
+    private final Version indexVersionCreated;
+
     private int storedFieldsReaderOrd = -1;
     private StoredFieldsReader storedFieldsReader = null;
 
@@ -77,6 +80,7 @@ final class LuceneChangesSnapshot implements Translog.Snapshot {
      * @param requiredFullRange if true, the snapshot will strictly check for the existence of operations between fromSeqNo and toSeqNo
      * @param singleConsumer    true if the snapshot is accessed by a single thread that creates the snapshot
      * @param accessStats       true if the stats of the snapshot can be accessed via {@link #totalOperations()}
+     * @param indexVersionCreated the version on which this index was created
      */
     LuceneChangesSnapshot(
         Engine.Searcher engineSearcher,
@@ -85,7 +89,8 @@ final class LuceneChangesSnapshot implements Translog.Snapshot {
         long toSeqNo,
         boolean requiredFullRange,
         boolean singleConsumer,
-        boolean accessStats
+        boolean accessStats,
+        Version indexVersionCreated
     ) throws IOException {
         if (fromSeqNo < 0 || toSeqNo < 0 || fromSeqNo > toSeqNo) {
             throw new IllegalArgumentException("Invalid range; from_seqno [" + fromSeqNo + "], to_seqno [" + toSeqNo + "]");
@@ -111,6 +116,7 @@ final class LuceneChangesSnapshot implements Translog.Snapshot {
         this.indexSearcher.setQueryCache(null);
         this.accessStats = accessStats;
         this.parallelArray = new ParallelArray(this.searchBatchSize);
+        this.indexVersionCreated = indexVersionCreated;
         final TopDocs topDocs = searchOperations(null, accessStats);
         this.totalHits = Math.toIntExact(topDocs.totalHits.value);
         this.scoreDocs = topDocs.scoreDocs;
@@ -272,21 +278,22 @@ final class LuceneChangesSnapshot implements Translog.Snapshot {
         return new IndexSearcher(Lucene.wrapAllDocsLive(engineSearcher.getDirectoryReader()));
     }
 
-    private static Query rangeQuery(long fromSeqNo, long toSeqNo) {
+    private static Query rangeQuery(long fromSeqNo, long toSeqNo, Version indexVersionCreated) {
         return new BooleanQuery.Builder().add(LongPoint.newRangeQuery(SeqNoFieldMapper.NAME, fromSeqNo, toSeqNo), BooleanClause.Occur.MUST)
-            .add(Queries.newNonNestedFilter(), BooleanClause.Occur.MUST) // exclude non-root nested documents
+            .add(Queries.newNonNestedFilter(indexVersionCreated), BooleanClause.Occur.MUST) // exclude non-root nested documents
             .build();
     }
 
-    static int countOperations(Engine.Searcher engineSearcher, long fromSeqNo, long toSeqNo) throws IOException {
+    static int countOperations(Engine.Searcher engineSearcher, long fromSeqNo, long toSeqNo, Version indexVersionCreated)
+        throws IOException {
         if (fromSeqNo < 0 || toSeqNo < 0 || fromSeqNo > toSeqNo) {
             throw new IllegalArgumentException("Invalid range; from_seqno [" + fromSeqNo + "], to_seqno [" + toSeqNo + "]");
         }
-        return newIndexSearcher(engineSearcher).count(rangeQuery(fromSeqNo, toSeqNo));
+        return newIndexSearcher(engineSearcher).count(rangeQuery(fromSeqNo, toSeqNo, indexVersionCreated));
     }
 
     private TopDocs searchOperations(FieldDoc after, boolean accurateTotalHits) throws IOException {
-        final Query rangeQuery = rangeQuery(Math.max(fromSeqNo, lastSeenSeqNo), toSeqNo);
+        final Query rangeQuery = rangeQuery(Math.max(fromSeqNo, lastSeenSeqNo), toSeqNo, indexVersionCreated);
         assert accurateTotalHits == false || after == null : "accurate total hits is required by the first batch only";
         final SortField sortBySeqNo = new SortField(SeqNoFieldMapper.NAME, SortField.Type.LONG);
         final TopFieldCollector collector = TopFieldCollector.create(

server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java

@@ -275,7 +275,7 @@ public class NestedQueryBuilder extends AbstractQueryBuilder<NestedQueryBuilder>
         Query innerQuery;
         NestedObjectMapper objectMapper = context.nestedScope().getObjectMapper();
         if (objectMapper == null) {
-            parentFilter = context.bitsetFilter(Queries.newNonNestedFilter());
+            parentFilter = context.bitsetFilter(Queries.newNonNestedFilter(context.indexVersionCreated()));
         } else {
             parentFilter = context.bitsetFilter(objectMapper.nestedTypeFilter());
         }
@@ -391,7 +391,7 @@ public class NestedQueryBuilder extends AbstractQueryBuilder<NestedQueryBuilder>
 
             Query rawParentFilter;
             if (parentObjectMapper == null) {
-                rawParentFilter = Queries.newNonNestedFilter();
+                rawParentFilter = Queries.newNonNestedFilter(context.getSearchExecutionContext().indexVersionCreated());
             } else {
                 rawParentFilter = parentObjectMapper.nestedTypeFilter();
             }

server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java

@@ -727,6 +727,6 @@ public class SearchExecutionContext extends QueryRewriteContext {
     }
 
     public NestedDocuments getNestedDocuments() {
-        return new NestedDocuments(mappingLookup, bitsetFilterCache::getBitSetProducer);
+        return new NestedDocuments(mappingLookup, bitsetFilterCache::getBitSetProducer, indexVersionCreated());
     }
 }

server/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java

@@ -29,6 +29,7 @@ import org.apache.lucene.util.BitSet;
 import org.apache.lucene.util.BitSetIterator;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.FixedBitSet;
+import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.routing.IndexRouting;
 import org.elasticsearch.common.lucene.search.Queries;
@@ -57,7 +58,7 @@ final class ShardSplittingQuery extends Query {
         this.indexMetadata = indexMetadata;
         this.indexRouting = IndexRouting.fromIndexMetadata(indexMetadata);
         this.shardId = shardId;
-        this.nestedParentBitSetProducer = hasNested ? newParentDocBitSetProducer() : null;
+        this.nestedParentBitSetProducer = hasNested ? newParentDocBitSetProducer(indexMetadata.getCreationVersion()) : null;
     }
 
     @Override
@@ -337,7 +338,7 @@ final class ShardSplittingQuery extends Query {
      * than once. There is no point in using BitsetFilterCache#BitSetProducerWarmer since we use this only as a delete by query which is
      * executed on a recovery-private index writer. There is no point in caching it and it won't have a cache hit either.
      */
-    private static BitSetProducer newParentDocBitSetProducer() {
-        return context -> BitsetFilterCache.bitsetFromQuery(Queries.newNonNestedFilter(), context);
+    private static BitSetProducer newParentDocBitSetProducer(Version indexCreationVersion) {
+        return context -> BitsetFilterCache.bitsetFromQuery(Queries.newNonNestedFilter(indexCreationVersion), context);
     }
 }

server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java

@@ -280,7 +280,7 @@ final class DefaultSearchContext extends SearchContext {
         if (nestedLookup != NestedLookup.EMPTY
             && nestedHelper.mightMatchNestedDocs(query)
             && (aliasFilter == null || nestedHelper.mightMatchNestedDocs(aliasFilter))) {
-            filters.add(Queries.newNonNestedFilter());
+            filters.add(Queries.newNonNestedFilter(searchExecutionContext.indexVersionCreated()));
         }
 
         if (aliasFilter != null) {

server/src/main/java/org/elasticsearch/search/NestedDocuments.java

@@ -18,6 +18,7 @@ import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.search.join.BitSetProducer;
 import org.apache.lucene.util.BitSet;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.index.mapper.MappingLookup;
 import org.elasticsearch.index.mapper.NestedLookup;
@@ -42,13 +43,14 @@ public class NestedDocuments {
      * Create a new NestedDocuments object for an index
      * @param mappingLookup     the index's mapping
      * @param filterProducer    a function to build BitSetProducers from filter queries
+     * @param indexVersionCreated the index creation version
      */
-    public NestedDocuments(MappingLookup mappingLookup, Function<Query, BitSetProducer> filterProducer) {
+    public NestedDocuments(MappingLookup mappingLookup, Function<Query, BitSetProducer> filterProducer, Version indexVersionCreated) {
         this.nestedLookup = mappingLookup.nestedLookup();
         if (this.nestedLookup == NestedLookup.EMPTY) {
             this.parentDocumentFilter = null;
         } else {
-            this.parentDocumentFilter = filterProducer.apply(Queries.newNonNestedFilter());
+            this.parentDocumentFilter = filterProducer.apply(Queries.newNonNestedFilter(indexVersionCreated));
             nestedLookup.getNestedParentFilters().forEach((k, v) -> parentObjectFilters.put(k, filterProducer.apply(v)));
             for (String nestedPath : nestedLookup.getNestedMappers().keySet()) {
                 childObjectFilters.put(nestedPath, null);

server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregator.java

@@ -60,7 +60,9 @@ public class NestedAggregator extends BucketsAggregator implements SingleBucketA
     ) throws IOException {
         super(name, factories, context, parent, cardinality, metadata);
 
-        Query parentFilter = parentObjectMapper != null ? parentObjectMapper.nestedTypeFilter() : Queries.newNonNestedFilter();
+        Query parentFilter = parentObjectMapper != null
+            ? parentObjectMapper.nestedTypeFilter()
+            : Queries.newNonNestedFilter(context.getIndexSettings().getIndexVersionCreated());
         this.parentFilter = context.bitsetFilterCache().getBitSetProducer(parentFilter);
         this.childFilter = childObjectMapper.nestedTypeFilter();
         this.collectsFromSingleBucket = cardinality.map(estimate -> estimate < 2);

server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregator.java

@@ -47,7 +47,7 @@ public class ReverseNestedAggregator extends BucketsAggregator implements Single
     ) throws IOException {
         super(name, factories, context, parent, cardinality, metadata);
         if (objectMapper == null) {
-            parentFilter = Queries.newNonNestedFilter();
+            parentFilter = Queries.newNonNestedFilter(context.getIndexSettings().getIndexVersionCreated());
         } else {
             parentFilter = objectMapper.nestedTypeFilter();
         }

server/src/main/java/org/elasticsearch/search/sort/SortBuilder.java

@@ -195,7 +195,7 @@ public abstract class SortBuilder<T extends SortBuilder<T>>
         final NestedObjectMapper objectMapper = context.nestedScope().getObjectMapper();
         final Query parentQuery;
         if (objectMapper == null) {
-            parentQuery = Queries.newNonNestedFilter();
+            parentQuery = Queries.newNonNestedFilter(context.indexVersionCreated());
         } else {
             parentQuery = objectMapper.nestedTypeFilter();
         }

server/src/test/java/org/elasticsearch/common/lucene/search/QueriesTests.java

@@ -14,6 +14,7 @@ import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.FieldExistsQuery;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.TermQuery;
+import org.elasticsearch.Version;
 import org.elasticsearch.index.mapper.SeqNoFieldMapper;
 import org.elasticsearch.test.ESTestCase;
 
@@ -21,9 +22,9 @@ public class QueriesTests extends ESTestCase {
 
     public void testNonNestedQuery() {
         // This is a custom query that extends AutomatonQuery and want to make sure the equals method works
-        assertEquals(Queries.newNonNestedFilter(), Queries.newNonNestedFilter());
-        assertEquals(Queries.newNonNestedFilter().hashCode(), Queries.newNonNestedFilter().hashCode());
-        assertEquals(Queries.newNonNestedFilter(), new FieldExistsQuery(SeqNoFieldMapper.PRIMARY_TERM_NAME));
+        assertEquals(Queries.newNonNestedFilter(Version.CURRENT), Queries.newNonNestedFilter(Version.CURRENT));
+        assertEquals(Queries.newNonNestedFilter(Version.CURRENT).hashCode(), Queries.newNonNestedFilter(Version.CURRENT).hashCode());
+        assertEquals(Queries.newNonNestedFilter(Version.CURRENT), new FieldExistsQuery(SeqNoFieldMapper.PRIMARY_TERM_NAME));
     }
 
     public void testIsNegativeQuery() {

server/src/test/java/org/elasticsearch/index/engine/LuceneChangesSnapshotTests.java

@@ -9,6 +9,7 @@
 package org.elasticsearch.index.engine;
 
 import org.apache.lucene.index.NoMergePolicy;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.IOUtils;
@@ -88,7 +89,8 @@ public class LuceneChangesSnapshotTests extends EngineTestCase {
                     toSeqNo,
                     false,
                     randomBoolean(),
-                    randomBoolean()
+                    randomBoolean(),
+                    Version.CURRENT
                 )
             ) {
                 searcher = null;
@@ -106,7 +108,8 @@ public class LuceneChangesSnapshotTests extends EngineTestCase {
                     toSeqNo,
                     true,
                     randomBoolean(),
-                    randomBoolean()
+                    randomBoolean(),
+                    Version.CURRENT
                 )
             ) {
                 searcher = null;
@@ -130,7 +133,8 @@ public class LuceneChangesSnapshotTests extends EngineTestCase {
                     toSeqNo,
                     false,
                     randomBoolean(),
-                    randomBoolean()
+                    randomBoolean(),
+                    Version.CURRENT
                 )
             ) {
                 searcher = null;
@@ -147,7 +151,8 @@ public class LuceneChangesSnapshotTests extends EngineTestCase {
                     toSeqNo,
                     true,
                     randomBoolean(),
-                    randomBoolean()
+                    randomBoolean(),
+                    Version.CURRENT
                 )
             ) {
                 searcher = null;
@@ -169,7 +174,8 @@ public class LuceneChangesSnapshotTests extends EngineTestCase {
                     toSeqNo,
                     true,
                     randomBoolean(),
-                    randomBoolean()
+                    randomBoolean(),
+                    Version.CURRENT
                 )
             ) {
                 searcher = null;
@@ -230,7 +236,8 @@ public class LuceneChangesSnapshotTests extends EngineTestCase {
                 maxSeqNo,
                 false,
                 randomBoolean(),
-                accessStats
+                accessStats,
+                Version.CURRENT
             )
         ) {
             if (accessStats) {

server/src/test/java/org/elasticsearch/index/mapper/NestedDocumentsTests.java

@@ -9,6 +9,7 @@
 package org.elasticsearch.index.mapper;
 
 import org.apache.lucene.search.join.QueryBitSetProducer;
+import org.elasticsearch.Version;
 import org.elasticsearch.search.LeafNestedDocuments;
 import org.elasticsearch.search.NestedDocuments;
 import org.elasticsearch.search.SearchHit;
@@ -45,7 +46,7 @@ public class NestedDocumentsTests extends MapperServiceTestCase {
         }));
 
         withLuceneIndex(mapperService, iw -> iw.addDocuments(doc.docs()), reader -> {
-            NestedDocuments nested = new NestedDocuments(mapperService.mappingLookup(), QueryBitSetProducer::new);
+            NestedDocuments nested = new NestedDocuments(mapperService.mappingLookup(), QueryBitSetProducer::new, Version.CURRENT);
             LeafNestedDocuments leaf = nested.getLeafNestedDocuments(reader.leaves().get(0));
 
             assertNotNull(leaf.advance(0));
@@ -142,7 +143,7 @@ public class NestedDocumentsTests extends MapperServiceTestCase {
         }));
 
         withLuceneIndex(mapperService, iw -> iw.addDocuments(doc.docs()), reader -> {
-            NestedDocuments nested = new NestedDocuments(mapperService.mappingLookup(), QueryBitSetProducer::new);
+            NestedDocuments nested = new NestedDocuments(mapperService.mappingLookup(), QueryBitSetProducer::new, Version.CURRENT);
             LeafNestedDocuments leaf = nested.getLeafNestedDocuments(reader.leaves().get(0));
 
             assertNotNull(leaf.advance(0));
@@ -257,7 +258,7 @@ public class NestedDocumentsTests extends MapperServiceTestCase {
         }));
 
         withLuceneIndex(mapperService, iw -> iw.addDocuments(doc.docs()), reader -> {
-            NestedDocuments nested = new NestedDocuments(mapperService.mappingLookup(), QueryBitSetProducer::new);
+            NestedDocuments nested = new NestedDocuments(mapperService.mappingLookup(), QueryBitSetProducer::new, Version.CURRENT);
             LeafNestedDocuments leaf = nested.getLeafNestedDocuments(reader.leaves().get(0));
 
             assertNotNull(leaf.advance(0));

server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java

@@ -817,7 +817,7 @@ public class NestedSortingTests extends AbstractFieldDataTestCase {
         IndexSearcher searcher
     ) throws IOException {
         Query query = new BooleanQuery.Builder().add(queryBuilder.toQuery(searchExecutionContext), Occur.MUST)
-            .add(Queries.newNonNestedFilter(), Occur.FILTER)
+            .add(Queries.newNonNestedFilter(searchExecutionContext.indexVersionCreated()), Occur.FILTER)
             .build();
         Sort sort = new Sort(sortBuilder.build(searchExecutionContext).field);
         return searcher.search(query, 10, sort);

server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java

@@ -106,6 +106,8 @@ public class DefaultSearchContextTests extends ESTestCase {
         IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY);
         when(indexService.getIndexSettings()).thenReturn(indexSettings);
         when(mapperService.getIndexSettings()).thenReturn(indexSettings);
+        when(searchExecutionContext.getIndexSettings()).thenReturn(indexSettings);
+        when(searchExecutionContext.indexVersionCreated()).thenReturn(indexSettings.getIndexVersionCreated());
 
         try (Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir)) {
 
@@ -273,7 +275,6 @@ public class DefaultSearchContextTests extends ESTestCase {
             context3.sliceBuilder(null).parsedQuery(parsedQuery).preProcess();
             assertEquals(context3.query(), context3.buildFilteredQuery(parsedQuery.query()));
 
-            when(searchExecutionContext.getIndexSettings()).thenReturn(indexSettings);
             when(searchExecutionContext.getFieldType(anyString())).thenReturn(mock(MappedFieldType.class));
 
             readerContext.close();

server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java

@@ -32,6 +32,7 @@ import org.apache.lucene.store.Directory;
 import org.apache.lucene.tests.index.RandomIndexWriter;
 import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
 import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.time.DateFormatter;
@@ -501,7 +502,7 @@ public class FiltersAggregatorTests extends AggregatorTestCase {
             assertThat(filters1.getBucketByKey("q1").getDocCount(), equalTo(1L));
         },
             new AggTestConfig(new FiltersAggregationBuilder("test", new KeyedFilter("q1", new TermQueryBuilder("author", "foo"))), ft)
-                .withQuery(Queries.newNonNestedFilter())
+                .withQuery(Queries.newNonNestedFilter(Version.CURRENT))
         );
         testCase(buildIndex, result -> {
             InternalFilters filters = (InternalFilters) result;
@@ -509,7 +510,7 @@ public class FiltersAggregatorTests extends AggregatorTestCase {
             assertThat(filters.getBucketByKey("q1").getDocCount(), equalTo(1L));
         },
             new AggTestConfig(new FiltersAggregationBuilder("test", new KeyedFilter("q1", new MatchAllQueryBuilder())), ft).withQuery(
-                Queries.newNonNestedFilter()
+                Queries.newNonNestedFilter(Version.CURRENT)
             )
         );
     }

server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java

@@ -373,7 +373,7 @@ public class NestedAggregatorTests extends AggregatorTestCase {
                 MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(VALUE_FIELD_NAME, NumberFieldMapper.NumberType.LONG);
 
                 BooleanQuery.Builder bq = new BooleanQuery.Builder();
-                bq.add(Queries.newNonNestedFilter(), BooleanClause.Occur.MUST);
+                bq.add(Queries.newNonNestedFilter(Version.CURRENT), BooleanClause.Occur.MUST);
                 bq.add(new TermQuery(new Term(IdFieldMapper.NAME, Uid.encodeId("2"))), BooleanClause.Occur.MUST_NOT);
 
                 InternalNested nested = searchAndReduce(
@@ -652,7 +652,9 @@ public class NestedAggregatorTests extends AggregatorTestCase {
 
                 Filter filter = searchAndReduce(
                     newSearcher(indexReader, false, true),
-                    new AggTestConfig(filterAggregationBuilder, fieldType1, fieldType2).withQuery(Queries.newNonNestedFilter())
+                    new AggTestConfig(filterAggregationBuilder, fieldType1, fieldType2).withQuery(
+                        Queries.newNonNestedFilter(Version.CURRENT)
+                    )
                 );
 
                 assertEquals("filterAgg", filter.getName());

server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java

@@ -34,6 +34,7 @@ import org.apache.lucene.store.Directory;
 import org.apache.lucene.tests.index.RandomIndexWriter;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.NumericUtils;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.lucene.search.Queries;
@@ -1480,7 +1481,7 @@ public class TermsAggregatorTests extends AggregatorTestCase {
                     // match root document only
                     StringTerms result = searchAndReduce(
                         newSearcher(indexReader, false, true),
-                        new AggTestConfig(terms, animalFieldType, nestedFieldType).withQuery(Queries.newNonNestedFilter())
+                        new AggTestConfig(terms, animalFieldType, nestedFieldType).withQuery(Queries.newNonNestedFilter(Version.CURRENT))
                     );
                     assertThat(result.getBuckets().get(0).getKeyAsString(), equalTo("pig"));
                     assertThat(result.getBuckets().get(0).docCount, equalTo(1L));

test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java

@@ -414,7 +414,9 @@ public abstract class AggregatorTestCase extends ESTestCase {
          */
         SearchExecutionContext subContext = spy(searchExecutionContext);
         MappingLookup disableNestedLookup = MappingLookup.fromMappers(Mapping.EMPTY, Set.of(), Set.of(), Set.of());
-        doReturn(new NestedDocuments(disableNestedLookup, bitsetFilterCache::getBitSetProducer)).when(subContext).getNestedDocuments();
+        doReturn(new NestedDocuments(disableNestedLookup, bitsetFilterCache::getBitSetProducer, indexSettings.getIndexVersionCreated()))
+            .when(subContext)
+            .getNestedDocuments();
         when(ctx.getSearchExecutionContext()).thenReturn(subContext);
 
         IndexShard indexShard = mock(IndexShard.class);

x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotRepository.java

@@ -173,7 +173,9 @@ public final class SourceOnlySnapshotRepository extends FilterRepository {
                     // do nothing;
                 }
             }, Store.OnClose.EMPTY);
-            Supplier<Query> querySupplier = mapperService.hasNested() ? Queries::newNestedFilter : null;
+            Supplier<Query> querySupplier = mapperService.hasNested()
+                ? () -> Queries.newNestedFilter(mapperService.getIndexSettings().getIndexVersionCreated())
+                : null;
             // SourceOnlySnapshot will take care of soft- and hard-deletes no special casing needed here
             SourceOnlySnapshot snapshot;
             snapshot = new SourceOnlySnapshot(overlayDir, querySupplier);

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/DocumentPermissions.java

@@ -174,10 +174,12 @@ public final class DocumentPermissions implements CacheKey {
                 if (nestedLookup != NestedLookup.EMPTY) {
                     NestedHelper nestedHelper = new NestedHelper(nestedLookup, context::isFieldMapped);
                     if (nestedHelper.mightMatchNestedDocs(roleQuery)) {
-                        roleQuery = new BooleanQuery.Builder().add(roleQuery, FILTER).add(Queries.newNonNestedFilter(), FILTER).build();
+                        roleQuery = new BooleanQuery.Builder().add(roleQuery, FILTER)
+                            .add(Queries.newNonNestedFilter(context.indexVersionCreated()), FILTER)
+                            .build();
                     }
                     // If access is allowed on root doc then also access is allowed on all nested docs of that root document:
-                    BitSetProducer rootDocs = context.bitsetFilter(Queries.newNonNestedFilter());
+                    BitSetProducer rootDocs = context.bitsetFilter(Queries.newNonNestedFilter(context.indexVersionCreated()));
                     ToChildBlockJoinQuery includeNestedDocs = new ToChildBlockJoinQuery(roleQuery, rootDocs);
                     filter.add(includeNestedDocs, SHOULD);
                 }

x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldMappingsIT.java

@@ -79,9 +79,9 @@ public class OldMappingsIT extends ESRestTestCase {
         String snapshotName = "snap";
         List<String> indices;
         if (oldVersion.before(Version.fromString("6.0.0"))) {
-            indices = Arrays.asList("filebeat", "winlogbeat", "custom");
+            indices = Arrays.asList("filebeat", "winlogbeat", "custom", "nested");
         } else {
-            indices = Arrays.asList("filebeat", "custom");
+            indices = Arrays.asList("filebeat", "custom", "nested");
         }
 
         int oldEsPort = Integer.parseInt(System.getProperty("tests.es.port"));
@@ -92,6 +92,7 @@ public class OldMappingsIT extends ESRestTestCase {
                 assertOK(oldEs.performRequest(createIndex("winlogbeat", "winlogbeat.json")));
             }
             assertOK(oldEs.performRequest(createIndex("custom", "custom.json")));
+            assertOK(oldEs.performRequest(createIndex("nested", "nested.json")));
 
             Request doc1 = new Request("PUT", "/" + "custom" + "/" + "doc" + "/" + "1");
             doc1.addParameter("refresh", "true");
@@ -122,6 +123,25 @@ public class OldMappingsIT extends ESRestTestCase {
             doc2.setJsonEntity(Strings.toString(bodyDoc2));
             assertOK(oldEs.performRequest(doc2));
 
+            Request doc3 = new Request("PUT", "/" + "nested" + "/" + "doc" + "/" + "1");
+            doc3.addParameter("refresh", "true");
+            XContentBuilder bodyDoc3 = XContentFactory.jsonBuilder()
+                .startObject()
+                .field("group", "fans")
+                .startArray("user")
+                .startObject()
+                .field("first", "John")
+                .field("last", "Smith")
+                .endObject()
+                .startObject()
+                .field("first", "Alice")
+                .field("last", "White")
+                .endObject()
+                .endArray()
+                .endObject();
+            doc3.setJsonEntity(Strings.toString(bodyDoc3));
+            assertOK(oldEs.performRequest(doc3));
+
             // register repo on old ES and take snapshot
             Request createRepoRequest = new Request("PUT", "/_snapshot/" + repoName);
             createRepoRequest.setJsonEntity(formatted("""
@@ -295,4 +315,47 @@ public class OldMappingsIT extends ESRestTestCase {
         assertEquals(List.of("some_value"), fields.get("completion"));
     }
 
+    public void testNestedDocuments() throws IOException {
+        Request search = new Request("POST", "/" + "nested" + "/_search");
+        Map<String, Object> response = entityAsMap(client().performRequest(search));
+        logger.info(response);
+        List<?> hits = (List<?>) (XContentMapValues.extractValue("hits.hits", response));
+        assertThat(hits, hasSize(1));
+        Map<?, ?> source = (Map<?, ?>) (XContentMapValues.extractValue("_source", (Map<?, ?>) hits.get(0)));
+        assertEquals("fans", source.get("group"));
+
+        search = new Request("POST", "/" + "nested" + "/_search");
+        XContentBuilder query = XContentBuilder.builder(XContentType.JSON.xContent())
+            .startObject()
+            .startObject("query")
+            .startObject("nested")
+            .field("path", "user")
+            .startObject("query")
+            .startObject("bool")
+            .startArray("must")
+            .startObject()
+            .startObject("match")
+            .field("user.first", "Alice")
+            .endObject()
+            .endObject()
+            .startObject()
+            .startObject("match")
+            .field("user.last", "White")
+            .endObject()
+            .endObject()
+            .endArray()
+            .endObject()
+            .endObject()
+            .endObject()
+            .endObject()
+            .endObject();
+        search.setJsonEntity(Strings.toString(query));
+        response = entityAsMap(client().performRequest(search));
+        logger.info(response);
+        hits = (List<?>) (XContentMapValues.extractValue("hits.hits", response));
+        assertThat(hits, hasSize(1));
+        source = (Map<?, ?>) (XContentMapValues.extractValue("_source", (Map<?, ?>) hits.get(0)));
+        assertEquals("fans", source.get("group"));
+    }
+
 }

x-pack/qa/repository-old-versions/src/test/resources/org/elasticsearch/oldrepos/nested.json

@@ -0,0 +1,7 @@
+"_default_": {
+  "properties": {
+    "user": {
+      "type": "nested"
+    }
+  }
+}