Speed counting filters/range/date_histogram aggs (#81322)

Lucene has a `Weight#count` method which will return a count of matching
documents if it can do so in constant time. We have similar code in the
`filters` aggregator and it seems silly to maintain two copies of the
same thing. Better yet, Lucene knows how to take the constant time count
path in a few cases that aggs don't and we get to pick those up. Namely,
we can now get a constant time count for `match_all` even when there are
deleted documents or document level security. We can also go the
constant time count path on cached queries now which is a lovel speed
bump.

Co-authored-by: Adrien Grand <jpountz@gmail.com> (seriously, he really rescued this thing)

docs/changelog/81322.yaml

@@ -0,0 +1,5 @@
+pr: 81322
+summary: Speed counting filters/range/date_histogram aggs
+area: Aggregations
+type: enhancement
+issues: []

server/src/internalClusterTest/java/org/elasticsearch/search/profile/aggregation/AggregationProfilerIT.java

@@ -698,7 +698,7 @@ public class AggregationProfilerIT extends ESIntegTestCase {
                                         "filters",
                                         matchesList().item(
                                             matchesMap().entry("query", "*:*")
-                                                .entry("results_from_metadata", 0)
+                                                .entry("segments_counted_in_constant_time", 0)
                                                 .entry("specialized_for", "match_all")
                                         )
                                     )

server/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java

@@ -141,6 +141,12 @@ public class IndicesQueryCache implements QueryCache, Closeable {
             return in.explain(context, doc);
         }
 
+        @Override
+        public int count(LeafReaderContext context) throws IOException {
+            shardKeyMap.add(context.reader());
+            return in.count(context);
+        }
+
         @Override
         public Scorer scorer(LeafReaderContext context) throws IOException {
             shardKeyMap.add(context.reader());

server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/DocValuesFieldExistsAdapter.java

@@ -1,60 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.search.aggregations.bucket.filter;
-
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.PointValues;
-import org.apache.lucene.search.DocValuesFieldExistsQuery;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.util.Bits;
-
-import java.io.IOException;
-import java.util.function.BiConsumer;
-
-/**
- * Specialized {@link QueryToFilterAdapter} for {@link DocValuesFieldExistsQuery} that reads counts from metadata.
- */
-class DocValuesFieldExistsAdapter extends QueryToFilterAdapter<DocValuesFieldExistsQuery> {
-    private int resultsFromMetadata;
-
-    DocValuesFieldExistsAdapter(IndexSearcher searcher, String key, DocValuesFieldExistsQuery query) {
-        super(searcher, key, query);
-    }
-
-    @Override
-    long count(LeafReaderContext ctx, FiltersAggregator.Counter counter, Bits live) throws IOException {
-        if (countCanUseMetadata(counter, live) && canCountFromMetadata(ctx)) {
-            resultsFromMetadata++;
-            PointValues points = ctx.reader().getPointValues(query().getField());
-            if (points == null) {
-                return 0;
-            }
-            return points.getDocCount();
-
-        }
-        return super.count(ctx, counter, live);
-    }
-
-    private boolean canCountFromMetadata(LeafReaderContext ctx) throws IOException {
-        FieldInfo info = ctx.reader().getFieldInfos().fieldInfo(query().getField());
-        if (info == null) {
-            // If we don't have any info then there aren't any values anyway.
-            return true;
-        }
-        return info.getPointDimensionCount() > 0;
-    }
-
-    @Override
-    void collectDebugInfo(BiConsumer<String, Object> add) {
-        super.collectDebugInfo(add);
-        add.accept("specialized_for", "docvalues_field_exists");
-        add.accept("results_from_metadata", resultsFromMetadata);
-    }
-}

server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/MatchAllQueryToFilterAdapter.java

@@ -12,7 +12,6 @@ import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.util.Bits;
 
 import java.io.IOException;
 import java.util.function.BiConsumer;
@@ -22,8 +21,6 @@ import java.util.function.IntPredicate;
  * Filter that matches every document.
  */
 class MatchAllQueryToFilterAdapter extends QueryToFilterAdapter<MatchAllDocsQuery> {
-    private int resultsFromMetadata;
-
     MatchAllQueryToFilterAdapter(IndexSearcher searcher, String key, MatchAllDocsQuery query) {
         super(searcher, key, query);
     }
@@ -38,19 +35,9 @@ class MatchAllQueryToFilterAdapter extends QueryToFilterAdapter<MatchAllDocsQuer
         return l -> true;
     }
 
-    @Override
-    long count(LeafReaderContext ctx, FiltersAggregator.Counter counter, Bits live) throws IOException {
-        if (countCanUseMetadata(counter, live)) {
-            resultsFromMetadata++;
-            return ctx.reader().maxDoc();  // TODO we could use numDocs even if live is not null because provides accurate numDocs.
-        }
-        return super.count(ctx, counter, live);
-    }
-
     @Override
     void collectDebugInfo(BiConsumer<String, Object> add) {
         super.collectDebugInfo(add);
         add.accept("specialized_for", "match_all");
-        add.accept("results_from_metadata", resultsFromMetadata);
     }
 }

server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/MatchNoneQueryToFilterAdapter.java

@@ -12,7 +12,6 @@ import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.util.Bits;
 
 import java.io.IOException;
 import java.util.function.BiConsumer;
@@ -36,11 +35,6 @@ class MatchNoneQueryToFilterAdapter extends QueryToFilterAdapter<MatchNoDocsQuer
         return l -> false;
     }
 
-    @Override
-    long count(LeafReaderContext ctx, FiltersAggregator.Counter counter, Bits live) throws IOException {
-        return 0;
-    }
-
     @Override
     void collectDebugInfo(BiConsumer<String, Object> add) {
         super.collectDebugInfo(add);

server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/MergedPointRangeQuery.java

@@ -103,6 +103,19 @@ public class MergedPointRangeQuery extends Query {
                 return true;
             }
 
+            @Override
+            public int count(LeafReaderContext context) throws IOException {
+                PointValues points = context.reader().getPointValues(field);
+                if (points == null) {
+                    return 0;
+                }
+                if (points.size() == points.getDocCount()) {
+                    // Each doc that has points has exactly one point.
+                    return singleValuedSegmentWeight().count(context);
+                }
+                return multiValuedSegmentWeight().count(context);
+            }
+
             @Override
             public Scorer scorer(LeafReaderContext context) throws IOException {
                 ScorerSupplier scorerSupplier = scorerSupplier(context);

server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/QueryToFilterAdapter.java

@@ -8,14 +8,12 @@
 
 package org.elasticsearch.search.aggregations.bucket.filter;
 
-import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.sandbox.search.IndexSortSortedNumericDocValuesRangeQuery;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.BulkScorer;
 import org.apache.lucene.search.ConstantScoreQuery;
-import org.apache.lucene.search.DocValuesFieldExistsQuery;
 import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.LeafCollector;
@@ -24,7 +22,6 @@ import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.PointRangeQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.ScoreMode;
-import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.util.Bits;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -60,12 +57,6 @@ public class QueryToFilterAdapter<Q extends Query> {
              */
             query = ((ConstantScoreQuery) query).getQuery();
         }
-        if (query instanceof TermQuery) {
-            return new TermQueryToFilterAdapter(searcher, key, (TermQuery) query);
-        }
-        if (query instanceof DocValuesFieldExistsQuery) {
-            return new DocValuesFieldExistsAdapter(searcher, key, (DocValuesFieldExistsQuery) query);
-        }
         if (query instanceof MatchAllDocsQuery) {
             return new MatchAllQueryToFilterAdapter(searcher, key, (MatchAllDocsQuery) query);
         }
@@ -83,6 +74,7 @@ public class QueryToFilterAdapter<Q extends Query> {
      * {@link #weight()} to build it when needed.
      */
     private Weight weight;
+    protected int segmentsCountedInConstantTime;
 
     QueryToFilterAdapter(IndexSearcher searcher, String key, Q query) {
         this.searcher = searcher;
@@ -124,31 +116,6 @@ public class QueryToFilterAdapter<Q extends Query> {
         return searcher;
     }
 
-    /**
-     * Would using index metadata like {@link IndexReader#docFreq}
-     * or {@link IndexReader#maxDoc} to count the number of matching documents
-     * produce the same answer as collecting the results with a sequence like
-     * {@code searcher.collect(counter); return counter.readAndReset();}?
-     */
-    protected static boolean countCanUseMetadata(FiltersAggregator.Counter counter, Bits live) {
-        if (live != null) {
-            /*
-             * We can only use metadata if all of the documents in the reader
-             * are visible. This is done by returning a null `live` bits. The
-             * name `live` is traditional because most of the time a non-null
-             * `live` bits means that there are deleted documents. But `live`
-             * might also be non-null if document level security is enabled.
-             */
-            return false;
-        }
-        /*
-         * We can only use metadata if we're not using the special docCount
-         * field. Otherwise we wouldn't know how many documents each lucene
-         * document represents.
-         */
-        return counter.docCount.alwaysOne();
-    }
-
     /**
      * Make a filter that matches this filter and the provided query.
      * <p>
@@ -223,6 +190,24 @@ public class QueryToFilterAdapter<Q extends Query> {
      * Count the number of documents that match this filter in a leaf.
      */
     long count(LeafReaderContext ctx, FiltersAggregator.Counter counter, Bits live) throws IOException {
+        /*
+         * weight().count will return the count of matches for ctx if it can do
+         * so in constant time, otherwise -1. The Weight is responsible for
+         * all of the cases where it can't return an accurate count *except*
+         * the doc_count field. That thing is ours, not Lucene's.
+         *
+         * For example, TermQuery will return -1 if there are deleted docs,
+         * otherwise it'll return number of documents with the term from the
+         * term statistics. MatchAllDocs will call `ctx.reader().numdocs()`
+         * to get the number of live docs.
+         */
+        if (counter.docCount.alwaysOne()) {
+            int count = weight().count(ctx);
+            if (count != -1) {
+                segmentsCountedInConstantTime++;
+                return count;
+            }
+        }
         BulkScorer scorer = weight().bulkScorer(ctx);
         if (scorer == null) {
             // No hits in this segment.
@@ -257,6 +242,7 @@ public class QueryToFilterAdapter<Q extends Query> {
      */
     void collectDebugInfo(BiConsumer<String, Object> add) {
         add.accept("query", query.toString());
+        add.accept("segments_counted_in_constant_time", segmentsCountedInConstantTime);
     }
 
     private Weight weight() throws IOException {

server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/TermQueryToFilterAdapter.java

@@ -1,44 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.search.aggregations.bucket.filter;
-
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.util.Bits;
-
-import java.io.IOException;
-import java.util.function.BiConsumer;
-
-/**
- * Specialized {@link QueryToFilterAdapter} for {@link TermQuery} that reads counts from metadata.
- */
-class TermQueryToFilterAdapter extends QueryToFilterAdapter<TermQuery> {
-    private int resultsFromMetadata;
-
-    TermQueryToFilterAdapter(IndexSearcher searcher, String key, TermQuery query) {
-        super(searcher, key, query);
-    }
-
-    @Override
-    long count(LeafReaderContext ctx, FiltersAggregator.Counter counter, Bits live) throws IOException {
-        if (countCanUseMetadata(counter, live)) {
-            resultsFromMetadata++;
-            return ctx.reader().docFreq(query().getTerm());
-        }
-        return super.count(ctx, counter, live);
-    }
-
-    @Override
-    void collectDebugInfo(BiConsumer<String, Object> add) {
-        super.collectDebugInfo(add);
-        add.accept("specialized_for", "term");
-        add.accept("results_from_metadata", resultsFromMetadata);
-    }
-}

server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java

@@ -112,7 +112,7 @@ public class IndicesQueryCacheTests extends ESTestCase {
         assertEquals(1L, stats.getCacheSize());
         assertEquals(1L, stats.getCacheCount());
         assertEquals(0L, stats.getHitCount());
-        assertEquals(1L, stats.getMissCount());
+        assertEquals(2L, stats.getMissCount());
 
         for (int i = 1; i < 20; ++i) {
             assertEquals(1, s.count(new DummyQuery(i)));
@@ -122,7 +122,7 @@ public class IndicesQueryCacheTests extends ESTestCase {
         assertEquals(10L, stats.getCacheSize());
         assertEquals(20L, stats.getCacheCount());
         assertEquals(0L, stats.getHitCount());
-        assertEquals(20L, stats.getMissCount());
+        assertEquals(40L, stats.getMissCount());
 
         s.count(new DummyQuery(10));
 
@@ -130,7 +130,7 @@ public class IndicesQueryCacheTests extends ESTestCase {
         assertEquals(10L, stats.getCacheSize());
         assertEquals(20L, stats.getCacheCount());
         assertEquals(1L, stats.getHitCount());
-        assertEquals(20L, stats.getMissCount());
+        assertEquals(40L, stats.getMissCount());
 
         IOUtils.close(r, dir);
 
@@ -139,7 +139,7 @@ public class IndicesQueryCacheTests extends ESTestCase {
         assertEquals(0L, stats.getCacheSize());
         assertEquals(20L, stats.getCacheCount());
         assertEquals(1L, stats.getHitCount());
-        assertEquals(20L, stats.getMissCount());
+        assertEquals(40L, stats.getMissCount());
 
         cache.onClose(shard);
 
@@ -188,7 +188,7 @@ public class IndicesQueryCacheTests extends ESTestCase {
         assertEquals(1L, stats1.getCacheSize());
         assertEquals(1L, stats1.getCacheCount());
         assertEquals(0L, stats1.getHitCount());
-        assertEquals(1L, stats1.getMissCount());
+        assertEquals(2L, stats1.getMissCount());
 
         QueryCacheStats stats2 = cache.getStats(shard2);
         assertEquals(0L, stats2.getCacheSize());
@@ -202,13 +202,13 @@ public class IndicesQueryCacheTests extends ESTestCase {
         assertEquals(1L, stats1.getCacheSize());
         assertEquals(1L, stats1.getCacheCount());
         assertEquals(0L, stats1.getHitCount());
-        assertEquals(1L, stats1.getMissCount());
+        assertEquals(2L, stats1.getMissCount());
 
         stats2 = cache.getStats(shard2);
         assertEquals(1L, stats2.getCacheSize());
         assertEquals(1L, stats2.getCacheCount());
         assertEquals(0L, stats2.getHitCount());
-        assertEquals(1L, stats2.getMissCount());
+        assertEquals(2L, stats2.getMissCount());
 
         for (int i = 0; i < 20; ++i) {
             assertEquals(1, s2.count(new DummyQuery(i)));
@@ -218,13 +218,13 @@ public class IndicesQueryCacheTests extends ESTestCase {
         assertEquals(0L, stats1.getCacheSize()); // evicted
         assertEquals(1L, stats1.getCacheCount());
         assertEquals(0L, stats1.getHitCount());
-        assertEquals(1L, stats1.getMissCount());
+        assertEquals(2L, stats1.getMissCount());
 
         stats2 = cache.getStats(shard2);
         assertEquals(10L, stats2.getCacheSize());
         assertEquals(20L, stats2.getCacheCount());
         assertEquals(1L, stats2.getHitCount());
-        assertEquals(20L, stats2.getMissCount());
+        assertEquals(40L, stats2.getMissCount());
 
         IOUtils.close(r1, dir1);
 
@@ -233,13 +233,13 @@ public class IndicesQueryCacheTests extends ESTestCase {
         assertEquals(0L, stats1.getCacheSize());
         assertEquals(1L, stats1.getCacheCount());
         assertEquals(0L, stats1.getHitCount());
-        assertEquals(1L, stats1.getMissCount());
+        assertEquals(2L, stats1.getMissCount());
 
         stats2 = cache.getStats(shard2);
         assertEquals(10L, stats2.getCacheSize());
         assertEquals(20L, stats2.getCacheCount());
         assertEquals(1L, stats2.getHitCount());
-        assertEquals(20L, stats2.getMissCount());
+        assertEquals(40L, stats2.getMissCount());
 
         cache.onClose(shard1);
 
@@ -254,7 +254,7 @@ public class IndicesQueryCacheTests extends ESTestCase {
         assertEquals(10L, stats2.getCacheSize());
         assertEquals(20L, stats2.getCacheCount());
         assertEquals(1L, stats2.getHitCount());
-        assertEquals(20L, stats2.getMissCount());
+        assertEquals(40L, stats2.getMissCount());
 
         IOUtils.close(r2, dir2);
         cache.onClose(shard2);

server/src/test/java/org/elasticsearch/indices/IndicesServiceCloseTests.java

@@ -9,6 +9,7 @@
 package org.elasticsearch.indices;
 
 import org.apache.lucene.document.LongPoint;
+import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings;
@@ -203,7 +204,7 @@ public class IndicesServiceCloseTests extends ESTestCase {
 
         Query query = LongPoint.newRangeQuery("foo", 0, 5);
         assertEquals(0L, cache.getStats(shard.shardId()).getCacheSize());
-        searcher.count(query);
+        searcher.search(new ConstantScoreQuery(query), 1);
         assertEquals(1L, cache.getStats(shard.shardId()).getCacheSize());
 
         searcher.close();
@@ -249,7 +250,7 @@ public class IndicesServiceCloseTests extends ESTestCase {
 
         Query query = LongPoint.newRangeQuery("foo", 0, 5);
         assertEquals(0L, cache.getStats(shard.shardId()).getCacheSize());
-        searcher.count(query);
+        searcher.search(new ConstantScoreQuery(query), 1);
         assertEquals(1L, cache.getStats(shard.shardId()).getCacheSize());
 
         searcher.close();

server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java

@@ -26,6 +26,7 @@ import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.QueryCachingPolicy;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.tests.index.RandomIndexWriter;
@@ -47,9 +48,11 @@ import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
 import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType;
 import org.elasticsearch.index.mapper.ObjectMapper;
+import org.elasticsearch.index.mapper.TextFieldMapper;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.ExistsQueryBuilder;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.index.query.MatchPhraseQueryBuilder;
 import org.elasticsearch.index.query.MatchQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
@@ -73,7 +76,9 @@ import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.Pipelin
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper;
 import org.elasticsearch.search.internal.ContextIndexSearcherTests.DocumentSubsetDirectoryReader;
+import org.elasticsearch.test.ListMatcher;
 import org.elasticsearch.test.MapMatcher;
+import org.hamcrest.Matcher;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -375,26 +380,104 @@ public class FiltersAggregatorTests extends AggregatorTestCase {
             "test",
             new KeyedFilter("q1", new RangeQueryBuilder("test").from("2020-01-01").to("2020-03-01").includeUpper(false))
         );
-        debugTestCase(builder, new MatchAllDocsQuery(), iw -> {
-            iw.addDocument(List.of(new LongPoint("test", DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2010-01-02"))));
-            iw.addDocument(List.of(new LongPoint("test", DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2020-01-02"))));
-        }, (InternalFilters filters, Class<? extends Aggregator> impl, Map<String, Map<String, Object>> debug) -> {
-            assertThat(filters.getBuckets(), hasSize(1));
-            assertThat(filters.getBucketByKey("q1").getDocCount(), equalTo(1L));
+        withIndex(iw -> {
+            iw.addDocuments(
+                List.of(
+                    List.of(new LongPoint("test", DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("1999-01-02"))),
+                    List.of(new LongPoint("test", DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2020-01-02")))
+                )
+            );
+        }, searcher -> {
+            debugTestCase(
+                builder,
+                new MatchAllDocsQuery(),
+                searcher,
+                (InternalFilters filters, Class<? extends Aggregator> impl, Map<String, Map<String, Object>> debug) -> {
+                    assertThat(filters.getBuckets(), hasSize(1));
+                    assertThat(filters.getBucketByKey("q1").getDocCount(), equalTo(1L));
+
+                    assertThat(impl, equalTo(FilterByFilterAggregator.class));
+                    assertMap(
+                        debug,
+                        matchesMap().entry(
+                            "test",
+                            matchesMap().entry("segments_with_doc_count_field", 0)
+                                .entry("segments_with_deleted_docs", 0)
+                                .entry("segments_collected", 0)
+                                .entry("segments_counted", greaterThanOrEqualTo(1))
+                                .entry(
+                                    "filters",
+                                    matchesList().item(
+                                        matchesMap().entry("query", "test:[1577836800000 TO 1583020799999]")
+                                            .entry("segments_counted_in_constant_time", 1)
+                                    )
+                                )
+                        )
+                    );
+                },
+                ft
+            );
+        });
+    }
 
-            assertThat(impl, equalTo(FilterByFilterAggregator.class));
-            assertMap(
-                debug,
-                matchesMap().entry(
-                    "test",
-                    matchesMap().entry("segments_with_doc_count_field", 0)
-                        .entry("segments_with_deleted_docs", 0)
-                        .entry("segments_collected", 0)
-                        .entry("segments_counted", greaterThanOrEqualTo(1))
-                        .entry("filters", matchesList().item(matchesMap().entry("query", "test:[1577836800000 TO 1583020799999]")))
+    /**
+     * Tests a filter that needs the cache to be fast.
+     */
+    public void testPhraseFilter() throws IOException {
+        MappedFieldType ft = new TextFieldMapper.TextFieldType("test");
+        AggregationBuilder builder = new FiltersAggregationBuilder(
+            "test",
+            new KeyedFilter("q1", new MatchPhraseQueryBuilder("test", "will find me").slop(0))
+        );
+        withIndex(iw -> {
+            iw.addDocuments(
+                List.of(
+                    List.of(new Field("test", "will not find me", TextFieldMapper.Defaults.FIELD_TYPE)),
+                    List.of(new Field("test", "will find me", TextFieldMapper.Defaults.FIELD_TYPE))
                 )
             );
-        }, ft);
+        }, searcher -> {
+            searcher.setQueryCachingPolicy(new QueryCachingPolicy() {
+                @Override
+                public boolean shouldCache(Query query) throws IOException {
+                    return true;
+                }
+
+                @Override
+                public void onUse(Query query) {}
+            });
+            for (Matcher<Integer> segmentsCountedInConstantTime : List.of(equalTo(0), greaterThanOrEqualTo(1))) {
+                debugTestCase(
+                    builder,
+                    new MatchAllDocsQuery(),
+                    searcher,
+                    (InternalFilters filters, Class<? extends Aggregator> impl, Map<String, Map<String, Object>> debug) -> {
+                        assertThat(filters.getBuckets(), hasSize(1));
+                        assertThat(filters.getBucketByKey("q1").getDocCount(), equalTo(1L));
+
+                        assertThat(impl, equalTo(FilterByFilterAggregator.class));
+                        assertMap(
+                            debug,
+                            matchesMap().entry(
+                                "test",
+                                matchesMap().entry("segments_with_doc_count_field", 0)
+                                    .entry("segments_with_deleted_docs", 0)
+                                    .entry("segments_collected", 0)
+                                    .entry("segments_counted", greaterThanOrEqualTo(1))
+                                    .entry(
+                                        "filters",
+                                        matchesList().item(
+                                            matchesMap().entry("query", "test:\"will find me\"")
+                                                .entry("segments_counted_in_constant_time", segmentsCountedInConstantTime)
+                                        )
+                                    )
+                            )
+                        );
+                    },
+                    ft
+                );
+            }
+        });
     }
 
     /**
@@ -459,7 +542,7 @@ public class FiltersAggregatorTests extends AggregatorTestCase {
                                 matchesList().item(
                                     matchesMap().entry("query", "*:*")
                                         .entry("specialized_for", "match_all")
-                                        .entry("results_from_metadata", greaterThanOrEqualTo(1))
+                                        .entry("segments_counted_in_constant_time", greaterThan(0))
                                 )
                             )
                     )
@@ -497,7 +580,7 @@ public class FiltersAggregatorTests extends AggregatorTestCase {
                                 matchesList().item(
                                     matchesMap().entry("query", "*:*")
                                         .entry("specialized_for", "match_all")
-                                        .entry("results_from_metadata", 0)
+                                        .entry("segments_counted_in_constant_time", 0)
                                 )
                             )
                     )
@@ -543,9 +626,8 @@ public class FiltersAggregatorTests extends AggregatorTestCase {
                         "test",
                         matchesMap().entry(
                             "filters",
-                            matchesList().item(
-                                matchesMap().entry("results_from_metadata", 0).entry("query", "a:0").entry("specialized_for", "term")
-                            ).item(matchesMap().entry("results_from_metadata", 0).entry("query", "a:1").entry("specialized_for", "term"))
+                            matchesList().item(matchesMap().entry("segments_counted_in_constant_time", 0).entry("query", "a:0"))
+                                .item(matchesMap().entry("segments_counted_in_constant_time", 0).entry("query", "a:1"))
                         )
                     )
                 );
@@ -558,6 +640,10 @@ public class FiltersAggregatorTests extends AggregatorTestCase {
      * This runs {@code filters} with a single {@code match_all} filter with
      * the index set up kind of like document level security. As a bonus, this
      * "looks" to the agg just like an index with deleted documents.
+     * <p>
+     * We actually <strong>do</strong> get to use the constant time counting
+     * here because {@code match_all} knows how to run constant time counts
+     * on filtered indices - that what {@link IndexReader#numDocs()} does.
      */
     public void testMatchAllOnFilteredIndex() throws IOException {
         AggregationBuilder builder = new FiltersAggregationBuilder("test", new KeyedFilter("q1", new MatchAllQueryBuilder()));
@@ -615,7 +701,9 @@ public class FiltersAggregatorTests extends AggregatorTestCase {
                         .entry(
                             "filters",
                             matchesList().item(
-                                matchesMap().entry("query", "*:*").entry("specialized_for", "match_all").entry("results_from_metadata", 0)
+                                matchesMap().entry("query", "*:*")
+                                    .entry("specialized_for", "match_all")
+                                    .entry("segments_counted_in_constant_time", 1)
                             )
                         )
                 );
@@ -623,6 +711,70 @@ public class FiltersAggregatorTests extends AggregatorTestCase {
         }
     }
 
+    /**
+     * This runs {@code filters} with a single {@code term} filter with
+     * the index set up kind of like document level security. As a bonus, this
+     * "looks" to the agg just like an index with deleted documents.
+     * <p>
+     * This can't use the constant time counting because {@code term} doesn't
+     * know how to count in constant time when there are deleted documents.
+     */
+    public void testTermOnFilteredIndex() throws IOException {
+        KeywordFieldType ft = new KeywordFieldType("foo");
+        AggregationBuilder builder = new FiltersAggregationBuilder("test", new KeyedFilter("q1", new TermQueryBuilder("foo", "bar")));
+        try (Directory directory = newDirectory()) {
+            RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
+            for (int i = 0; i < 10; i++) {
+                indexWriter.addDocument(List.of(new Field("foo", "bar", KeywordFieldMapper.Defaults.FIELD_TYPE), new LongPoint("t", i)));
+            }
+            indexWriter.close();
+
+            try (DirectoryReader directoryReader = DirectoryReader.open(directory)) {
+                BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(createIndexSettings(), new BitsetFilterCache.Listener() {
+                    @Override
+                    public void onRemoval(ShardId shardId, Accountable accountable) {}
+
+                    @Override
+                    public void onCache(ShardId shardId, Accountable accountable) {}
+                });
+                IndexReader limitedReader = new DocumentSubsetDirectoryReader(
+                    ElasticsearchDirectoryReader.wrap(directoryReader, new ShardId(bitsetFilterCache.index(), 0)),
+                    bitsetFilterCache,
+                    LongPoint.newRangeQuery("t", 5, Long.MAX_VALUE)
+                );
+                IndexSearcher searcher = newIndexSearcher(limitedReader);
+                AggregationContext context = createAggregationContext(searcher, new MatchAllDocsQuery(), ft);
+                FilterByFilterAggregator aggregator = createAggregator(builder, context);
+                aggregator.preCollection();
+                searcher.search(context.query(), aggregator);
+                aggregator.postCollection();
+
+                InternalAggregation result = aggregator.buildTopLevel();
+                result = result.reduce(
+                    List.of(result),
+                    new AggregationReduceContext.ForFinal(context.bigArrays(), getMockScriptService(), () -> false, null, b -> {})
+                );
+                InternalFilters filters = (InternalFilters) result;
+                assertThat(filters.getBuckets(), hasSize(1));
+                assertThat(filters.getBucketByKey("q1").getDocCount(), equalTo(5L));
+
+                Map<String, Object> debug = new HashMap<>();
+                aggregator.collectDebugInfo(debug::put);
+                assertMap(
+                    debug,
+                    matchesMap().entry("segments_counted", greaterThanOrEqualTo(1))
+                        .entry("segments_collected", 0)
+                        .entry("segments_with_doc_count_field", 0)
+                        .entry("segments_with_deleted_docs", 0)
+                        .entry(
+                            "filters",
+                            matchesList().item(matchesMap().entry("query", "foo:bar").entry("segments_counted_in_constant_time", 0))
+                        )
+                );
+            }
+        }
+    }
+
     public void testComplexUnionDisabledFilterByFilter() throws IOException {
         MappedFieldType dft = new DateFieldMapper.DateFieldType(
             "date",
@@ -664,7 +816,13 @@ public class FiltersAggregatorTests extends AggregatorTestCase {
                 debug,
                 matchesMap().entry(
                     "test",
-                    matchesMap().entry("filters", matchesList().item(matchesMap().entry("query", "date:[1577836800000 TO 1583020799999]")))
+                    matchesMap().entry(
+                        "filters",
+                        matchesList().item(
+                            matchesMap().entry("query", "date:[1577836800000 TO 1583020799999]")
+                                .entry("segments_counted_in_constant_time", 0)
+                        )
+                    )
                 )
             );
         }, dft, kft);
@@ -699,6 +857,7 @@ public class FiltersAggregatorTests extends AggregatorTestCase {
                                 matchesList().item(
                                     matchesMap().entry("query", "MatchNoDocsQuery(\"User requested \"match_none\" query.\")")
                                         .entry("specialized_for", "match_none")
+                                        .entry("segments_counted_in_constant_time", greaterThan(0))
                                 )
                             )
                     )
@@ -736,6 +895,7 @@ public class FiltersAggregatorTests extends AggregatorTestCase {
                                 matchesList().item(
                                     matchesMap().entry("query", "MatchNoDocsQuery(\"User requested \"match_none\" query.\")")
                                         .entry("specialized_for", "match_none")
+                                        .entry("segments_counted_in_constant_time", greaterThan(0))
                                 )
                             )
                     )
@@ -773,9 +933,7 @@ public class FiltersAggregatorTests extends AggregatorTestCase {
                             .entry(
                                 "filters",
                                 matchesList().item(
-                                    matchesMap().entry("query", "f:0")
-                                        .entry("specialized_for", "term")
-                                        .entry("results_from_metadata", greaterThan(0))
+                                    matchesMap().entry("query", "f:0").entry("segments_counted_in_constant_time", greaterThan(0))
                                 )
                             )
                     )
@@ -814,9 +972,7 @@ public class FiltersAggregatorTests extends AggregatorTestCase {
                             .entry(
                                 "filters",
                                 matchesList().item(
-                                    matchesMap().entry("query", "f:0")
-                                        .entry("specialized_for", "term")
-                                        .entry("results_from_metadata", greaterThan(0))
+                                    matchesMap().entry("query", "f:0").entry("segments_counted_in_constant_time", greaterThan(0))
                                 )
                             )
                     )
@@ -877,7 +1033,7 @@ public class FiltersAggregatorTests extends AggregatorTestCase {
                                     matchesMap().entry(
                                         "query",
                                         "MergedPointRange[+test:[1262390400000 TO 1262390405000] +test:[1262390400000 TO 1262390409000]]"
-                                    )
+                                    ).entry("segments_counted_in_constant_time", greaterThanOrEqualTo(1))
                                 )
                             )
                     )
@@ -936,7 +1092,7 @@ public class FiltersAggregatorTests extends AggregatorTestCase {
                                     matchesMap().entry(
                                         "query",
                                         "MergedPointRange[+test:[1262390400000 TO 1262390405000] +test:[1262390400000 TO 1262390409000]]"
-                                    )
+                                    ).entry("segments_counted_in_constant_time", greaterThanOrEqualTo(1))
                                 )
                             )
                     )
@@ -998,7 +1154,7 @@ public class FiltersAggregatorTests extends AggregatorTestCase {
                                     matchesMap().entry(
                                         "query",
                                         "MergedPointRange[+test:[1262390400000 TO 1262390405000] +test:[1262390400000 TO 1262390409000]]"
-                                    )
+                                    ).entry("segments_counted_in_constant_time", greaterThanOrEqualTo(1))
                                 )
                             )
                     )
@@ -1074,6 +1230,9 @@ public class FiltersAggregatorTests extends AggregatorTestCase {
                 assertThat(sum.value(), equalTo(15.0));
 
                 assertThat(impl, equalTo(FilterByFilterAggregator.class));
+                ListMatcher filtersMatcher = matchesList().item(
+                    matchesMap().entry("query", "test:[1262304000000 TO 1267401599999]").entry("segments_counted_in_constant_time", 0)
+                ).item(matchesMap().entry("query", "test:[1577836800000 TO 1583020799999]").entry("segments_counted_in_constant_time", 0));
                 assertMap(
                     debug,
                     matchesMap().entry(
@@ -1082,11 +1241,7 @@ public class FiltersAggregatorTests extends AggregatorTestCase {
                             .entry("segments_with_deleted_docs", 0)
                             .entry("segments_collected", greaterThanOrEqualTo(1))
                             .entry("segments_counted", 0)
-                            .entry(
-                                "filters",
-                                matchesList().item(matchesMap().entry("query", "test:[1262304000000 TO 1267401599999]"))
-                                    .item(matchesMap().entry("query", "test:[1577836800000 TO 1583020799999]"))
-                            )
+                            .entry("filters", filtersMatcher)
                     ).entry("test.s", matchesMap()).entry("test.m", matchesMap())
                 );
             },
@@ -1320,7 +1475,7 @@ public class FiltersAggregatorTests extends AggregatorTestCase {
 
     public void testDocValuesFieldExistsForKeyword() throws IOException {
         KeywordFieldMapper.KeywordFieldType ft = new KeywordFieldMapper.KeywordFieldType("f", true, true, Map.of());
-        docValuesFieldExistsTestCase(new ExistsQueryBuilder("f"), ft, false, i -> {
+        docValuesFieldExistsTestCase(new ExistsQueryBuilder("f"), ft, true, i -> {
             BytesRef text = new BytesRef(randomAlphaOfLength(5));
             return List.of(new Field("f", text, KeywordFieldMapper.Defaults.FIELD_TYPE), new SortedSetDocValuesField("f", text));
         });
@@ -1333,7 +1488,7 @@ public class FiltersAggregatorTests extends AggregatorTestCase {
     private void docValuesFieldExistsTestCase(
         QueryBuilder exists,
         MappedFieldType fieldType,
-        boolean canUseMetadata,
+        boolean countsResultsInConstantTime,
         IntFunction<Iterable<? extends IndexableField>> buildDocWithField
     ) throws IOException {
         AggregationBuilder builder = new FiltersAggregationBuilder("test", new KeyedFilter("q1", exists));
@@ -1355,8 +1510,7 @@ public class FiltersAggregatorTests extends AggregatorTestCase {
 
             assertThat(impl, equalTo(FilterByFilterAggregator.class));
             MapMatcher expectedFilterDebug = matchesMap().extraOk()
-                .entry("specialized_for", "docvalues_field_exists")
-                .entry("results_from_metadata", canUseMetadata ? greaterThan(0) : equalTo(0));
+                .entry("segments_counted_in_constant_time", countsResultsInConstantTime ? greaterThan(0) : equalTo(0));
             assertMap(debug, matchesMap().entry("test", matchesMap().extraOk().entry("filters", matchesList().item(expectedFilterDebug))));
         }, fieldType, fnft);
     }
@@ -1375,10 +1529,7 @@ public class FiltersAggregatorTests extends AggregatorTestCase {
             assertThat(aggregator, instanceOf(FilterByFilterAggregator.class));
 
             Map<String, Object> debug = collectAndGetFilterDebugInfo(searcher, aggregator);
-            assertMap(
-                debug,
-                matchesMap().extraOk().entry("specialized_for", "docvalues_field_exists").entry("results_from_metadata", greaterThan(0))
-            );
+            assertMap(debug, matchesMap().extraOk().entry("segments_counted_in_constant_time", greaterThan(0)));
         }, fieldType, fnft);
         testCase(builder, new MatchAllDocsQuery(), buildIndex, (InternalFilters result) -> {
             assertThat(result.getBuckets(), hasSize(1));

server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java

@@ -878,8 +878,7 @@ public class DateHistogramAggregatorTests extends DateHistogramAggregatorTestCas
                                                 "filters",
                                                 matchesList().item(
                                                     matchesMap().entry("query", "FieldExistsQuery [field=f]")
-                                                        .entry("specialized_for", "docvalues_field_exists")
-                                                        .entry("results_from_metadata", greaterThan(0))
+                                                        .entry("segments_counted_in_constant_time", greaterThan(0))
                                                 )
                                             )
                                     )
@@ -939,7 +938,7 @@ public class DateHistogramAggregatorTests extends DateHistogramAggregatorTestCas
                                                 matchesList().item(
                                                     matchesMap().entry("query", "*:*")
                                                         .entry("specialized_for", "match_all")
-                                                        .entry("results_from_metadata", 0)
+                                                        .entry("segments_counted_in_constant_time", 0)
                                                 )
                                             )
                                     )