Support synthetic _source for _doc_count field (#91465)

This add synthetic `_source` support for the `_doc_count` field so
downsampling should play nicely with sythetic `_source`.

docs/changelog/91465.yaml

@@ -0,0 +1,5 @@
+pr: 91465
+summary: Support synthetic `_source` for `_doc_count` field
+area: TSDB
+type: enhancement
+issues: []

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureMetaFieldMapper.java

@@ -11,6 +11,7 @@ package org.elasticsearch.index.mapper.extras;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
+import org.elasticsearch.index.mapper.SourceLoader;
 import org.elasticsearch.index.mapper.TextSearchInfo;
 import org.elasticsearch.index.mapper.ValueFetcher;
 import org.elasticsearch.index.query.SearchExecutionContext;
@@ -68,4 +69,8 @@ public class RankFeatureMetaFieldMapper extends MetadataFieldMapper {
         return CONTENT_TYPE;
     }
 
+    @Override
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+        return SourceLoader.SyntheticFieldLoader.NOTHING;
+    }
 }

plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java

@@ -16,6 +16,7 @@ import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
 import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType;
 import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType;
+import org.elasticsearch.index.mapper.SourceLoader;
 import org.elasticsearch.index.mapper.ValueFetcher;
 import org.elasticsearch.index.query.SearchExecutionContext;
 
@@ -96,4 +97,9 @@ public class SizeFieldMapper extends MetadataFieldMapper {
     public FieldMapper.Builder getMergeBuilder() {
         return new Builder().init(this);
     }
+
+    @Override
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+        return SourceLoader.SyntheticFieldLoader.NOTHING;
+    }
 }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml

@@ -584,6 +584,45 @@ _source filtering:
         kwd: foo
   - is_false: fields
 
+---
+_doc_count:
+  - skip:
+      version: " - 8.5.99"
+      reason: introduced in 8.6.0
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            number_of_replicas: 0
+          mappings:
+            _source:
+              mode: synthetic
+
+  - do:
+      index:
+        index:   test
+        id:      1
+        refresh: true
+        body:
+          _doc_count: 3
+          foo: bar
+
+  - do:
+      get:
+        index: test
+        id:    1
+  - match: {_index: "test"}
+  - match: {_id: "1"}
+  - match: {_version: 1}
+  - match: {found: true}
+  - match:
+      _source:
+        _doc_count: 3
+        foo: bar
+  - is_false: fields
+
 ---
 ip with ignore_malformed:
   - skip:

server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java

@@ -30,6 +30,7 @@ import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.DocumentParserContext;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
+import org.elasticsearch.index.mapper.SourceLoader;
 import org.elasticsearch.index.mapper.TimeSeriesParams;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
@@ -718,6 +719,11 @@ public class FieldCapabilitiesIT extends ESIntegTestCase {
             return CONTENT_TYPE;
         }
 
+        @Override
+        public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+            throw new UnsupportedOperationException();
+        }
+
         private static final TypeParser PARSER = new FixedTypeParser(c -> new TestMetadataMapper());
     }
 }

server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java

@@ -268,4 +268,9 @@ public class DataStreamTimestampFieldMapper extends MetadataFieldMapper {
     public boolean isEnabled() {
         return enabled;
     }
+
+    @Override
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+        return SourceLoader.SyntheticFieldLoader.NOTHING;
+    }
 }

server/src/main/java/org/elasticsearch/index/mapper/DocCountFieldMapper.java

@@ -8,14 +8,20 @@
 package org.elasticsearch.index.mapper;
 
 import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.PostingsEnum;
+import org.apache.lucene.index.Term;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.xcontent.XContentParserUtils;
 import org.elasticsearch.index.query.QueryShardException;
 import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.Collections;
+import java.util.Map;
+import java.util.stream.Stream;
 
 /** Mapper for the doc_count field. */
 public class DocCountFieldMapper extends MetadataFieldMapper {
@@ -25,6 +31,11 @@ public class DocCountFieldMapper extends MetadataFieldMapper {
 
     private static final DocCountFieldMapper INSTANCE = new DocCountFieldMapper();
 
+    /**
+     * The term that is the key to the postings list that stores the doc counts.
+     */
+    private static final Term TERM = new Term(NAME, NAME);
+
     public static final TypeParser PARSER = new FixedTypeParser(c -> INSTANCE);
 
     public static final class DocCountFieldType extends MappedFieldType {
@@ -115,4 +126,49 @@ public class DocCountFieldMapper extends MetadataFieldMapper {
     public static IndexableField field(int count) {
         return new CustomTermFreqField(NAME, NAME, count);
     }
+
+    @Override
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+        return new SyntheticFieldLoader();
+    }
+
+    /**
+     * The lookup for loading values.
+     */
+    public static PostingsEnum leafLookup(LeafReader reader) throws IOException {
+        return reader.postings(TERM);
+    }
+
+    private class SyntheticFieldLoader implements SourceLoader.SyntheticFieldLoader {
+        private PostingsEnum postings;
+        private boolean hasValue;
+
+        @Override
+        public Stream<Map.Entry<String, StoredFieldLoader>> storedFieldLoaders() {
+            return Stream.empty();
+        }
+
+        @Override
+        public DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf) throws IOException {
+            postings = leafLookup(leafReader);
+            if (postings == null) {
+                hasValue = false;
+                return null;
+            }
+            return docId -> hasValue = docId == postings.advance(docId);
+        }
+
+        @Override
+        public boolean hasValue() {
+            return hasValue;
+        }
+
+        @Override
+        public void write(XContentBuilder b) throws IOException {
+            if (hasValue == false) {
+                return;
+            }
+            b.field(NAME, postings.freq());
+        }
+    }
 }

server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java

@@ -196,4 +196,8 @@ public class FieldNamesFieldMapper extends MetadataFieldMapper {
         return CONTENT_TYPE;
     }
 
+    @Override
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+        return SourceLoader.SyntheticFieldLoader.NOTHING;
+    }
 }

server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java

@@ -41,6 +41,11 @@ public abstract class IdFieldMapper extends MetadataFieldMapper {
         return CONTENT_TYPE;
     }
 
+    @Override
+    public final SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+        return SourceLoader.SyntheticFieldLoader.NOTHING;
+    }
+
     /**
      * Description of the document being parsed used in error messages. Not
      * called unless there is an error.

server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java

@@ -88,4 +88,8 @@ public final class IgnoredFieldMapper extends MetadataFieldMapper {
         return CONTENT_TYPE;
     }
 
+    @Override
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+        return SourceLoader.SyntheticFieldLoader.NOTHING;
+    }
 }

server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java

@@ -102,4 +102,9 @@ public class IndexFieldMapper extends MetadataFieldMapper {
     protected String contentType() {
         return CONTENT_TYPE;
     }
+
+    @Override
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+        return SourceLoader.SyntheticFieldLoader.NOTHING;
+    }
 }

server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java

@@ -106,4 +106,9 @@ public class LegacyTypeFieldMapper extends MetadataFieldMapper {
     protected String contentType() {
         return CONTENT_TYPE;
     }
+
+    @Override
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+        return SourceLoader.SyntheticFieldLoader.NOTHING;
+    }
 }

server/src/main/java/org/elasticsearch/index/mapper/MapperBuilderContext.java

@@ -11,6 +11,7 @@ package org.elasticsearch.index.mapper;
 import org.elasticsearch.common.Strings;
 
 import java.util.Objects;
+import java.util.function.BooleanSupplier;
 
 /**
  * Holds context for building Mapper objects from their Builders
@@ -21,19 +22,28 @@ public final class MapperBuilderContext {
      * The root context, to be used when building a tree of mappers
      */
     public static MapperBuilderContext root(boolean isSourceSynthetic) {
-        return new MapperBuilderContext(isSourceSynthetic);
+        return new MapperBuilderContext(null, () -> isSourceSynthetic);
+    }
+
+    /**
+     * A context to use to build metadata fields.
+     */
+    public static MapperBuilderContext forMetadata() {
+        return new MapperBuilderContext(
+            null,
+            () -> { throw new UnsupportedOperationException("metadata fields can't check if _source is synthetic"); }
+        );
     }
 
     private final String path;
-    private final boolean isSourceSynthetic;
+    private final BooleanSupplier isSourceSynthetic;
 
-    private MapperBuilderContext(boolean isSourceSynthetic) {
-        this.path = null;
-        this.isSourceSynthetic = isSourceSynthetic;
+    MapperBuilderContext(String path, boolean isSourceSynthetic) {
+        this(Objects.requireNonNull(path), () -> isSourceSynthetic);
     }
 
-    MapperBuilderContext(String path, boolean isSourceSynthetic) {
-        this.path = Objects.requireNonNull(path);
+    private MapperBuilderContext(String path, BooleanSupplier isSourceSynthetic) {
+        this.path = path;
         this.isSourceSynthetic = isSourceSynthetic;
     }
 
@@ -56,7 +66,10 @@ public final class MapperBuilderContext {
         return path + "." + name;
     }
 
+    /**
+     * Is the {@code _source} field being reconstructed on the fly?
+     */
     public boolean isSourceSynthetic() {
-        return isSourceSynthetic;
+        return isSourceSynthetic.getAsBoolean();
     }
 }

server/src/main/java/org/elasticsearch/index/mapper/Mapping.java

@@ -124,6 +124,10 @@ public final class Mapping implements ToXContentFragment {
         return sfm != null && sfm.isSynthetic();
     }
 
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+        return root.syntheticFieldLoader(Arrays.stream(metadataMappers));
+    }
+
     /**
      * Merges a new mapping into the existing one.
      *

server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java

@@ -120,7 +120,7 @@ public final class MappingParser {
                 @SuppressWarnings("unchecked")
                 Map<String, Object> fieldNodeMap = (Map<String, Object>) fieldNode;
                 MetadataFieldMapper metadataFieldMapper = typeParser.parse(fieldName, fieldNodeMap, parserContext)
-                    .build(MapperBuilderContext.root(false));
+                    .build(MapperBuilderContext.forMetadata());
                 metadataMappers.put(metadataFieldMapper.getClass(), metadataFieldMapper);
                 fieldNodeMap.remove("type");
                 checkNoRemainingFields(fieldName, fieldNodeMap);

server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java

@@ -155,7 +155,7 @@ public abstract class MetadataFieldMapper extends FieldMapper {
     @Override
     protected void parseCreateField(DocumentParserContext context) throws IOException {
         throw new MapperParsingException(
-            "Field [" + name() + "] is a metadata field and cannot be added inside" + " a document. Use the index API request parameters."
+            "Field [" + name() + "] is a metadata field and cannot be added inside a document. Use the index API request parameters."
         );
     }
 
@@ -173,4 +173,6 @@ public abstract class MetadataFieldMapper extends FieldMapper {
         // do nothing
     }
 
+    @Override
+    public abstract SourceLoader.SyntheticFieldLoader syntheticFieldLoader();
 }

server/src/main/java/org/elasticsearch/index/mapper/NestedPathFieldMapper.java

@@ -97,4 +97,8 @@ public class NestedPathFieldMapper extends MetadataFieldMapper {
         return NAME;
     }
 
+    @Override
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+        return SourceLoader.SyntheticFieldLoader.NOTHING;
+    }
 }

server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java

@@ -578,11 +578,9 @@ public class ObjectMapper extends Mapper implements Cloneable {
 
     }
 
-    @Override
-    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader(Stream<Mapper> extra) {
         return new SyntheticSourceFieldLoader(
-            mappers.values()
-                .stream()
+            Stream.concat(extra, mappers.values().stream())
                 .sorted(Comparator.comparing(Mapper::name))
                 .map(Mapper::syntheticFieldLoader)
                 .filter(l -> l != null)
@@ -590,6 +588,11 @@ public class ObjectMapper extends Mapper implements Cloneable {
         );
     }
 
+    @Override
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+        return syntheticFieldLoader(Stream.empty());
+    }
+
     private class SyntheticSourceFieldLoader implements SourceLoader.SyntheticFieldLoader {
         private final List<SourceLoader.SyntheticFieldLoader> fields;
         private boolean hasValue;

server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java

@@ -131,4 +131,8 @@ public class RoutingFieldMapper extends MetadataFieldMapper {
         return CONTENT_TYPE;
     }
 
+    @Override
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+        return SourceLoader.SyntheticFieldLoader.NOTHING;
+    }
 }

server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java

@@ -239,4 +239,8 @@ public class SeqNoFieldMapper extends MetadataFieldMapper {
         return CONTENT_TYPE;
     }
 
+    @Override
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+        return SourceLoader.SyntheticFieldLoader.NOTHING;
+    }
 }

server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java

@@ -270,4 +270,9 @@ public class SourceFieldMapper extends MetadataFieldMapper {
     public boolean isSynthetic() {
         return mode == Mode.SYNTHETIC;
     }
+
+    @Override
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+        return SourceLoader.SyntheticFieldLoader.NOTHING;
+    }
 }

server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java

@@ -89,7 +89,7 @@ public interface SourceLoader {
         private final Map<String, SyntheticFieldLoader.StoredFieldLoader> storedFieldLoaders;
 
         public Synthetic(Mapping mapping) {
-            loader = mapping.getRoot().syntheticFieldLoader();
+            loader = mapping.syntheticFieldLoader();
             storedFieldLoaders = Map.copyOf(loader.storedFieldLoaders().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)));
         }
 

server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java

@@ -155,6 +155,11 @@ public class TimeSeriesIdFieldMapper extends MetadataFieldMapper {
         return CONTENT_TYPE;
     }
 
+    @Override
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+        return SourceLoader.SyntheticFieldLoader.NOTHING;
+    }
+
     /**
      * Decode the {@code _tsid} into a human readable map.
      */

server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java

@@ -92,4 +92,9 @@ public class VersionFieldMapper extends MetadataFieldMapper {
     protected String contentType() {
         return CONTENT_TYPE;
     }
+
+    @Override
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+        return SourceLoader.SyntheticFieldLoader.NOTHING;
+    }
 }

server/src/main/java/org/elasticsearch/search/aggregations/bucket/DocCountProvider.java

@@ -10,7 +10,6 @@ package org.elasticsearch.search.aggregations.bucket;
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.PostingsEnum;
-import org.apache.lucene.index.Term;
 import org.elasticsearch.index.mapper.DocCountFieldMapper;
 
 import java.io.IOException;
@@ -41,7 +40,7 @@ public class DocCountProvider {
     }
 
     public void setLeafReaderContext(LeafReaderContext ctx) throws IOException {
-        docCountPostings = ctx.reader().postings(new Term(DocCountFieldMapper.NAME, DocCountFieldMapper.NAME));
+        docCountPostings = DocCountFieldMapper.leafLookup(ctx.reader());
     }
 
     public boolean alwaysOne() {

server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java

@@ -8,8 +8,16 @@
 package org.elasticsearch.index.mapper;
 
 import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.LeafReaderContext;
+import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader;
+import org.elasticsearch.index.fieldvisitor.StoredFieldLoader;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.stream.IntStream;
 
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
 
 public class DocCountFieldMapperTests extends MapperServiceTestCase {
 
@@ -57,4 +65,35 @@ public class DocCountFieldMapperTests extends MapperServiceTestCase {
         Exception e = expectThrows(MapperParsingException.class, () -> mapper.parse(source(b -> b.array(CONTENT_TYPE, 10, 20, 30))));
         assertThat(e.getCause().getMessage(), containsString("Arrays are not allowed for field [_doc_count]."));
     }
+
+    public void testSyntheticSource() throws IOException {
+        DocumentMapper mapper = createDocumentMapper(syntheticSourceMapping(b -> {}));
+        assertThat(syntheticSource(mapper, b -> b.field(CONTENT_TYPE, 10)), equalTo("{\"_doc_count\":10}"));
+    }
+
+    public void testSyntheticSourceMany() throws IOException {
+        MapperService mapper = createMapperService(syntheticSourceMapping(b -> {}));
+        List<Integer> counts = randomList(2, 10000, () -> between(1, Integer.MAX_VALUE));
+        withLuceneIndex(mapper, iw -> {
+            for (int c : counts) {
+                iw.addDocument(mapper.documentMapper().parse(source(b -> b.field(CONTENT_TYPE, c))).rootDoc());
+            }
+        }, reader -> {
+            int i = 0;
+            SourceLoader loader = mapper.mappingLookup().newSourceLoader();
+            assertTrue(loader.requiredStoredFields().isEmpty());
+            for (LeafReaderContext leaf : reader.leaves()) {
+                int[] docIds = IntStream.range(0, leaf.reader().maxDoc()).toArray();
+                SourceLoader.Leaf sourceLoaderLeaf = loader.leaf(leaf.reader(), docIds);
+                LeafStoredFieldLoader storedFieldLoader = StoredFieldLoader.empty().getLoader(leaf, docIds);
+                for (int docId : docIds) {
+                    assertThat(
+                        "doc " + docId,
+                        sourceLoaderLeaf.source(storedFieldLoader, docId).utf8ToString(),
+                        equalTo("{\"_doc_count\":" + counts.get(i++) + "}")
+                    );
+                }
+            }
+        });
+    }
 }

server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java

@@ -2573,6 +2573,11 @@ public class DocumentParserTests extends MapperServiceTestCase {
                 return CONTENT_TYPE;
             }
 
+            @Override
+            public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+                throw new UnsupportedOperationException();
+            }
+
             private static final TypeParser PARSER = new FixedTypeParser(c -> new MockMetadataMapper());
         }
 

server/src/test/java/org/elasticsearch/index/mapper/ExternalMetadataMapper.java

@@ -31,6 +31,10 @@ public class ExternalMetadataMapper extends MetadataFieldMapper {
         context.doc().add(new StringField(FIELD_NAME, FIELD_VALUE, Store.YES));
     }
 
-    public static final TypeParser PARSER = new FixedTypeParser(c -> new ExternalMetadataMapper());
+    @Override
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+        throw new UnsupportedOperationException();
+    }
 
+    public static final TypeParser PARSER = new FixedTypeParser(c -> new ExternalMetadataMapper());
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldMapper.java

@@ -17,6 +17,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.mapper.ConstantFieldType;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
+import org.elasticsearch.index.mapper.SourceLoader;
 import org.elasticsearch.index.mapper.ValueFetcher;
 import org.elasticsearch.index.query.SearchExecutionContext;
 
@@ -109,4 +110,9 @@ public class DataTierFieldMapper extends MetadataFieldMapper {
     protected String contentType() {
         return CONTENT_TYPE;
     }
+
+    @Override
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+        return SourceLoader.SyntheticFieldLoader.NOTHING;
+    }
 }