Add `synthetic_source` support to `aggregate_metric_double` fields (#88909)

This PR implements synthetic_source support to the aggregate_metric_double
field type

Relates to #86603

docs/changelog/88909.yaml

@@ -0,0 +1,5 @@
+pr: 88909
+summary: Add `synthetic_source` support to `aggregate_metric_double` fields
+area: Mapping
+type: enhancement
+issues: []

docs/reference/mapping/fields/synthetic-source.asciidoc

@@ -28,6 +28,7 @@ space. There are a couple of restrictions to be aware of:
 * Synthetic `_source` can be used with indices that contain only these field
 types:
 
+** <<aggregate-metric-double-synthetic-source, `aggregate_metric_double`>>
 ** <<boolean-synthetic-source,`boolean`>>
 ** <<numeric-synthetic-source,`byte`>>
 ** <<numeric-synthetic-source,`double`>>

docs/reference/mapping/types/aggregate-metric-double.asciidoc

@@ -251,3 +251,55 @@ The search returns the following hit. The value of the `default_metric` field,
 }
 ----
 // TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,/]
+
+ifeval::["{release-state}"=="unreleased"]
+[[aggregate-metric-double-synthetic-source]]
+==== Synthetic source
+`aggregate_metric-double` fields support <<synthetic-source,synthetic `_source`>> in their default
+configuration. Synthetic `_source` cannot be used together with <<ignore-malformed,`ignore_malformed`>>.
+
+For example:
+[source,console,id=synthetic-source-aggregate-metric-double-example]
+----
+PUT idx
+{
+  "mappings": {
+    "_source": { "mode": "synthetic" },
+    "properties": {
+      "agg_metric": {
+        "type": "aggregate_metric_double",
+        "metrics": [ "min", "max", "sum", "value_count" ],
+        "default_metric": "max"
+      }
+    }
+  }
+}
+
+PUT idx/_doc/1
+{
+  "agg_metric": {
+    "min": -302.50,
+    "max": 702.30,
+    "sum": 200.0,
+    "value_count": 25
+  }
+}
+----
+// TEST[s/$/\nGET idx\/_doc\/1?filter_path=_source\n/]
+
+Will become:
+
+[source,console-result]
+----
+{
+  "agg_metric": {
+    "min": -302.50,
+    "max": 702.30,
+    "sum": 200.0,
+    "value_count": 25
+  }
+}
+----
+// TEST[s/^/{"_source":/ s/\n$/}/]
+
+endif::[]

server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java

@@ -1719,7 +1719,7 @@ public class NumberFieldMapper extends FieldMapper {
 
         @Override
         public Leaf leaf(LeafReader reader, int[] docIdsInLeaf) throws IOException {
-            SortedNumericDocValues dv = dv(reader);
+            SortedNumericDocValues dv = docValuesOrNull(reader, name);
             if (dv == null) {
                 return SourceLoader.SyntheticFieldLoader.NOTHING_LEAF;
             }
@@ -1830,12 +1830,12 @@ public class NumberFieldMapper extends FieldMapper {
          * an "empty" implementation if there aren't any doc values. We need to be able to
          * tell if there aren't any and return our empty leaf source loader.
          */
-        private SortedNumericDocValues dv(LeafReader reader) throws IOException {
-            SortedNumericDocValues dv = reader.getSortedNumericDocValues(name);
+        public static SortedNumericDocValues docValuesOrNull(LeafReader reader, String fieldName) throws IOException {
+            SortedNumericDocValues dv = reader.getSortedNumericDocValues(fieldName);
             if (dv != null) {
                 return dv;
             }
-            NumericDocValues single = reader.getNumericDocValues(name);
+            NumericDocValues single = reader.getNumericDocValues(fieldName);
             if (single != null) {
                 return DocValues.singleton(single);
             }

test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java

@@ -253,6 +253,13 @@ public abstract class MapperTestCase extends MapperServiceTestCase {
         return true;
     }
 
+    /**
+     * Override to disable testing {@code copy_to} in fields that don't support it.
+     */
+    protected boolean supportsCopyTo() {
+        return true;
+    }
+
     protected void metaMapping(XContentBuilder b) throws IOException {
         minimalMapping(b);
     }
@@ -893,15 +900,17 @@ public abstract class MapperTestCase extends MapperServiceTestCase {
 
     public final void testSyntheticSourceInvalid() throws IOException {
         List<SyntheticSourceInvalidExample> examples = new ArrayList<>(syntheticSourceSupport().invalidExample());
-        examples.add(
-            new SyntheticSourceInvalidExample(
-                matchesPattern("field \\[field] of type \\[.+] doesn't support synthetic source because it declares copy_to"),
-                b -> {
-                    syntheticSourceSupport().example(5).mapping().accept(b);
-                    b.field("copy_to", "bar");
-                }
-            )
-        );
+        if (supportsCopyTo()) {
+            examples.add(
+                new SyntheticSourceInvalidExample(
+                    matchesPattern("field \\[field] of type \\[.+] doesn't support synthetic source because it declares copy_to"),
+                    b -> {
+                        syntheticSourceSupport().example(5).mapping().accept(b);
+                        b.field("copy_to", "bar");
+                    }
+                )
+            );
+        }
         for (SyntheticSourceInvalidExample example : examples) {
             Exception e = expectThrows(
                 IllegalArgumentException.class,

x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java

@@ -9,6 +9,7 @@ package org.elasticsearch.xpack.aggregatemetric.mapper;
 import org.apache.lucene.index.DocValues;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.search.Query;
@@ -32,6 +33,7 @@ import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperBuilderContext;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
 import org.elasticsearch.index.mapper.SimpleMappedFieldType;
+import org.elasticsearch.index.mapper.SourceLoader;
 import org.elasticsearch.index.mapper.SourceValueFetcher;
 import org.elasticsearch.index.mapper.TextSearchInfo;
 import org.elasticsearch.index.mapper.TimeSeriesParams;
@@ -574,7 +576,6 @@ public class AggregateDoubleMetricFieldMapper extends FieldMapper {
 
     @Override
     protected void parseCreateField(DocumentParserContext context) throws IOException {
-
         context.path().add(simpleName());
         XContentParser.Token token;
         XContentSubParser subParser = null;
@@ -675,4 +676,95 @@ public class AggregateDoubleMetricFieldMapper extends FieldMapper {
     public FieldMapper.Builder getMergeBuilder() {
         return new Builder(simpleName(), ignoreMalformedByDefault, indexCreatedVersion).metric(metricType).init(this);
     }
+
+    @Override
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+        if (ignoreMalformed) {
+            throw new IllegalArgumentException(
+                "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it ignores malformed numbers"
+            );
+        }
+        return new AggregateMetricSyntheticFieldLoader(name(), simpleName(), metrics);
+    }
+
+    public static class AggregateMetricSyntheticFieldLoader implements SourceLoader.SyntheticFieldLoader {
+        private final String name;
+        private final String simpleName;
+        private final EnumSet<Metric> metrics;
+
+        protected AggregateMetricSyntheticFieldLoader(String name, String simpleName, EnumSet<Metric> metrics) {
+            this.name = name;
+            this.simpleName = simpleName;
+            this.metrics = metrics;
+        }
+
+        @Override
+        public Leaf leaf(LeafReader reader, int[] docIdsInLeaf) throws IOException {
+            Map<Metric, SortedNumericDocValues> metricDocValues = new EnumMap<>(Metric.class);
+            for (Metric m : metrics) {
+                String fieldName = subfieldName(name, m);
+                SortedNumericDocValues dv = NumberFieldMapper.NumericSyntheticFieldLoader.docValuesOrNull(reader, fieldName);
+                if (dv != null) {
+                    metricDocValues.put(m, dv);
+                }
+            }
+
+            if (metricDocValues.isEmpty()) {
+                return SourceLoader.SyntheticFieldLoader.NOTHING_LEAF;
+            }
+
+            return new AggregateMetricSyntheticFieldLoader.ImmediateLeaf(metricDocValues);
+        }
+
+        private class ImmediateLeaf implements Leaf {
+            private final Map<Metric, SortedNumericDocValues> metricDocValues;
+            private final Set<Metric> metricHasValue = EnumSet.noneOf(Metric.class);
+
+            ImmediateLeaf(Map<Metric, SortedNumericDocValues> metricDocValues) {
+                assert metricDocValues.isEmpty() == false : "doc_values for metrics cannot be empty";
+                this.metricDocValues = metricDocValues;
+            }
+
+            @Override
+            public boolean empty() {
+                return false;
+            }
+
+            @Override
+            public boolean advanceToDoc(int docId) throws IOException {
+                // It is required that all defined metrics must exist. In this case
+                // it is enough to check for the first docValue. However, in the future
+                // we may relax the requirement of all metrics existing. In this case
+                // we should check the doc value for each metric separately
+                metricHasValue.clear();
+                for (Map.Entry<Metric, SortedNumericDocValues> e : metricDocValues.entrySet()) {
+                    if (e.getValue().advanceExact(docId)) {
+                        metricHasValue.add(e.getKey());
+                    }
+                }
+
+                return metricHasValue.isEmpty() == false;
+            }
+
+            @Override
+            public void write(XContentBuilder b) throws IOException {
+                if (metricHasValue.isEmpty()) {
+                    return;
+                }
+                b.startObject(simpleName);
+                for (Map.Entry<Metric, SortedNumericDocValues> entry : metricDocValues.entrySet()) {
+                    if (metricHasValue.contains(entry.getKey())) {
+                        String metricName = entry.getKey().name();
+                        long value = entry.getValue().nextValue();
+                        if (entry.getKey() == Metric.value_count) {
+                            b.field(metricName, value);
+                        } else {
+                            b.field(metricName, NumericUtils.sortableLongToDouble(value));
+                        }
+                    }
+                }
+                b.endObject();
+            }
+        }
+    }
 }

x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java

@@ -20,12 +20,16 @@ import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xpack.aggregatemetric.AggregateMetricMapperPlugin;
+import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.Metric;
 import org.hamcrest.Matchers;
 import org.junit.AssumptionViolatedException;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.Collection;
+import java.util.EnumSet;
 import java.util.Iterator;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -33,6 +37,7 @@ import static org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetr
 import static org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.Names.METRICS;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.matchesPattern;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.core.IsInstanceOf.instanceOf;
@@ -393,7 +398,7 @@ public class AggregateDoubleMetricFieldMapperTests extends MapperTestCase {
 
         Mapper fieldMapper = mapper.mappers().getMapper("field");
         assertThat(fieldMapper, instanceOf(AggregateDoubleMetricFieldMapper.class));
-        assertEquals(AggregateDoubleMetricFieldMapper.Metric.sum, ((AggregateDoubleMetricFieldMapper) fieldMapper).defaultMetric());
+        assertEquals(Metric.sum, ((AggregateDoubleMetricFieldMapper) fieldMapper).defaultMetric());
     }
 
     /**
@@ -406,7 +411,7 @@ public class AggregateDoubleMetricFieldMapperTests extends MapperTestCase {
 
         Mapper fieldMapper = mapper.mappers().getMapper("field");
         assertThat(fieldMapper, instanceOf(AggregateDoubleMetricFieldMapper.class));
-        assertEquals(AggregateDoubleMetricFieldMapper.Metric.value_count, ((AggregateDoubleMetricFieldMapper) fieldMapper).defaultMetric);
+        assertEquals(Metric.value_count, ((AggregateDoubleMetricFieldMapper) fieldMapper).defaultMetric);
     }
 
     /**
@@ -416,7 +421,7 @@ public class AggregateDoubleMetricFieldMapperTests extends MapperTestCase {
         DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping));
         Mapper fieldMapper = mapper.mappers().getMapper("field");
         assertThat(fieldMapper, instanceOf(AggregateDoubleMetricFieldMapper.class));
-        assertEquals(AggregateDoubleMetricFieldMapper.Metric.max, ((AggregateDoubleMetricFieldMapper) fieldMapper).defaultMetric);
+        assertEquals(Metric.max, ((AggregateDoubleMetricFieldMapper) fieldMapper).defaultMetric);
     }
 
     /**
@@ -505,8 +510,8 @@ public class AggregateDoubleMetricFieldMapperTests extends MapperTestCase {
      *  subfields of aggregate_metric_double should not be searchable or exposed in field_caps
      */
     public void testNoSubFieldsIterated() throws IOException {
-        AggregateDoubleMetricFieldMapper.Metric[] values = AggregateDoubleMetricFieldMapper.Metric.values();
-        List<AggregateDoubleMetricFieldMapper.Metric> subset = randomSubsetOf(randomIntBetween(1, values.length), values);
+        Metric[] values = Metric.values();
+        List<Metric> subset = randomSubsetOf(randomIntBetween(1, values.length), values);
         DocumentMapper mapper = createDocumentMapper(
             fieldMapping(b -> b.field("type", CONTENT_TYPE).field(METRICS_FIELD, subset).field(DEFAULT_METRIC, subset.get(0)))
         );
@@ -589,11 +594,58 @@ public class AggregateDoubleMetricFieldMapperTests extends MapperTestCase {
 
     @Override
     protected SyntheticSourceSupport syntheticSourceSupport() {
-        throw new AssumptionViolatedException("not supported");
+        return new AggregateDoubleMetricSyntheticSourceSupport();
     }
 
     @Override
     protected IngestScriptSupport ingestScriptSupport() {
         throw new AssumptionViolatedException("not supported");
     }
+
+    protected final class AggregateDoubleMetricSyntheticSourceSupport implements SyntheticSourceSupport {
+
+        private final EnumSet<Metric> storedMetrics = EnumSet.copyOf(randomNonEmptySubsetOf(Arrays.asList(Metric.values())));
+
+        @Override
+        public SyntheticSourceExample example(int maxVals) {
+            // aggregate_metric_double field does not support arrays
+            Map<String, Object> value = randomAggregateMetric();
+            return new SyntheticSourceExample(value, value, this::mapping);
+        }
+
+        private Map<String, Object> randomAggregateMetric() {
+            Map<String, Object> value = new LinkedHashMap<>(storedMetrics.size());
+            for (Metric m : storedMetrics) {
+                if (Metric.value_count == m) {
+                    value.put(m.name(), randomLongBetween(1, 1_000_000));
+                } else {
+                    value.put(m.name(), randomDouble());
+                }
+            }
+            return value;
+        }
+
+        private void mapping(XContentBuilder b) throws IOException {
+            String[] metrics = storedMetrics.stream().map(Metric::toString).toArray(String[]::new);
+            b.field("type", CONTENT_TYPE).array(METRICS_FIELD, metrics).field(DEFAULT_METRIC, metrics[0]);
+        }
+
+        @Override
+        public List<SyntheticSourceInvalidExample> invalidExample() throws IOException {
+            return List.of(
+                new SyntheticSourceInvalidExample(
+                    matchesPattern("field \\[field] of type \\[.+] doesn't support synthetic source because it ignores malformed numbers"),
+                    b -> {
+                        mapping(b);
+                        b.field("ignore_malformed", true);
+                    }
+                )
+            );
+        }
+    }
+
+    @Override
+    protected boolean supportsCopyTo() {
+        return false;
+    }
 }

x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/100_synthetic_source.yml

@@ -0,0 +1,53 @@
+constant_keyword:
+  - skip:
+      version: " - 8.4.99"
+      reason: synthetic source support added  in 8.5.0
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              metric:
+                type: aggregate_metric_double
+                metrics: [min, max, value_count]
+                default_metric: max
+
+  - do:
+      index:
+        index:  test
+        id:     "1"
+        refresh: false # Do not refresh on every insert so that we get both docs in the same segment
+        body:
+          metric:
+            min: 18.2
+            max: 100
+            value_count: 50
+
+  - do:
+      index:
+        index:  test
+        id:     "2"
+        refresh: true
+        body:
+          metric:
+            min: 10.0
+            max: 20.0
+            value_count: 5
+
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            ids:
+              values: [1, 2]
+  - match:
+      hits.hits.0._source:
+        metric:
+          min: 18.2
+          max: 100.0
+          value_count: 50