Enforce synthetic source for time series indices (#93380)

Support for synthetic source is also added to `unsigned_long` field as part of this change.
This is required because `unsigned_long` field types can be used in tsdb indices and
this change would prohibit the usage of these field type otherwise.

Closes #92319

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml

@@ -413,3 +413,124 @@ nested fields:
                           rx:
                             type: long
                             time_series_metric: gauge
+
+---
+regular source:
+  - skip:
+      version: " - 8.6.99"
+      reason: synthetic source
+
+  - do:
+      catch: '/time series indices only support synthetic source/'
+      indices.create:
+        index: tsdb_index
+        body:
+          settings:
+            index:
+              mode: time_series
+              routing_path: [k8s.pod.uid]
+              time_series:
+                start_time: 2021-04-28T00:00:00Z
+                end_time: 2021-04-29T00:00:00Z
+          mappings:
+            _source:
+              mode: stored
+            properties:
+              "@timestamp":
+                type: date
+              k8s:
+                properties:
+                  pod:
+                    properties:
+                      uid:
+                        type: keyword
+                        time_series_dimension: true
+---
+disabled source:
+  - skip:
+      version: " - 8.6.99"
+      reason: synthetic source
+
+  - do:
+      catch: '/time series indices only support synthetic source/'
+      indices.create:
+        index: tsdb_index
+        body:
+          settings:
+            index:
+              mode: time_series
+              routing_path: [k8s.pod.uid]
+              time_series:
+                start_time: 2021-04-28T00:00:00Z
+                end_time: 2021-04-29T00:00:00Z
+          mappings:
+            _source:
+              mode: disabled
+            properties:
+              "@timestamp":
+                type: date
+              k8s:
+                properties:
+                  pod:
+                    properties:
+                      uid:
+                        type: keyword
+                        time_series_dimension: true
+
+---
+source include/exclude:
+  - skip:
+      version: " - 8.6.99"
+      reason: synthetic source
+
+  - do:
+      catch: '/filtering the stored _source is incompatible with synthetic source/'
+      indices.create:
+        index: tsdb_index
+        body:
+          settings:
+            index:
+              mode: time_series
+              routing_path: [k8s.pod.uid]
+              time_series:
+                start_time: 2021-04-28T00:00:00Z
+                end_time: 2021-04-29T00:00:00Z
+          mappings:
+            _source:
+              includes: [a]
+            properties:
+              "@timestamp":
+                type: date
+              k8s:
+                properties:
+                  pod:
+                    properties:
+                      uid:
+                        type: keyword
+                        time_series_dimension: true
+
+  - do:
+      catch: '/filtering the stored _source is incompatible with synthetic source/'
+      indices.create:
+        index: tsdb_index
+        body:
+          settings:
+            index:
+              mode: time_series
+              routing_path: [k8s.pod.uid]
+              time_series:
+                start_time: 2021-04-28T00:00:00Z
+                end_time: 2021-04-29T00:00:00Z
+          mappings:
+            _source:
+              excludes: [b]
+            properties:
+              "@timestamp":
+                type: date
+              k8s:
+                properties:
+                  pod:
+                    properties:
+                      uid:
+                        type: keyword
+                        time_series_dimension: true

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml

@@ -148,94 +148,3 @@ clone:
 
   - match: {hits.total.value: 1}
   - match: {hits.hits.0.fields._tsid: [{k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507, metricset: pod}]}
-
----
-clone no source index:
-  - skip:
-      version: " - 8.1.99"
-      reason: tsdb indexing changed in 8.2.0
-
-  - do:
-      indices.create:
-        index: test_no_source
-        body:
-          settings:
-            index:
-              mode: time_series
-              routing_path: [ metricset, k8s.pod.uid ]
-              time_series:
-                start_time: 2021-04-28T00:00:00Z
-                end_time: 2021-04-29T00:00:00Z
-              number_of_shards: 1
-              number_of_replicas: 0
-          mappings:
-            _source:
-              enabled: false
-            properties:
-              "@timestamp":
-                type: date
-              metricset:
-                type: keyword
-                time_series_dimension: true
-              k8s:
-                properties:
-                  pod:
-                    properties:
-                      uid:
-                        type: keyword
-                        time_series_dimension: true
-                      name:
-                        type: keyword
-                      ip:
-                        type: ip
-                      network:
-                        properties:
-                          tx:
-                            type: long
-                          rx:
-                            type: long
-
-  - do:
-      bulk:
-        refresh: true
-        index: test_no_source
-        body:
-          - '{"index": {}}'
-          - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}'
-          - '{"index": {}}'
-          - '{"@timestamp": "2021-04-28T18:50:24.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2005177954, "rx": 801479970}}}}'
-          - '{"index": {}}'
-          - '{"@timestamp": "2021-04-28T18:50:44.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2006223737, "rx": 802337279}}}}'
-          - '{"index": {}}'
-          - '{"@timestamp": "2021-04-28T18:51:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.2", "network": {"tx": 2012916202, "rx": 803685721}}}}'
-          - '{"index": {}}'
-          - '{"@timestamp": "2021-04-28T18:50:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434521831, "rx": 530575198}}}}'
-          - '{"index": {}}'
-          - '{"@timestamp": "2021-04-28T18:50:23.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434577921, "rx": 530600088}}}}'
-          - '{"index": {}}'
-          - '{"@timestamp": "2021-04-28T18:50:53.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434587694, "rx": 530604797}}}}'
-          - '{"index": {}}'
-          - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434595272, "rx": 530605511}}}}'
-
-  - do:
-      indices.put_settings:
-        index: test_no_source
-        body:
-          index.blocks.write: true
-
-  - do:
-      indices.clone:
-        index: test_no_source
-        target: test_no_source_clone
-
-  - do:
-      search:
-        index: test_no_source_clone
-        body:
-          docvalue_fields:
-            - field: _tsid
-          query:
-            query_string:
-              query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat'
-  - match: {hits.total.value: 1}
-  - match: {hits.hits.0.fields._tsid: [{k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507, metricset: pod}]}

server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java

@@ -10,6 +10,7 @@ package org.elasticsearch.search.fieldcaps;
 
 import org.apache.http.entity.ContentType;
 import org.apache.http.entity.StringEntity;
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.fieldcaps.FieldCapabilities;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesAction;
@@ -37,6 +38,7 @@ import org.elasticsearch.index.mapper.DocumentParserContext;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
 import org.elasticsearch.index.mapper.SourceLoader;
+import org.elasticsearch.index.mapper.StringStoredFieldFieldLoader;
 import org.elasticsearch.index.mapper.TimeSeriesParams;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
@@ -118,6 +120,7 @@ public class FieldCapabilitiesIT extends ESIntegTestCase {
             .endObject()
             .startObject("playlist")
             .field("type", "text")
+            .field("store", true)
             .endObject()
             .startObject("some_dimension")
             .field("type", "keyword")
@@ -845,7 +848,13 @@ public class FieldCapabilitiesIT extends ESIntegTestCase {
 
         @Override
         public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
-            throw new UnsupportedOperationException();
+            return new StringStoredFieldFieldLoader(name(), simpleName(), null) {
+                @Override
+                protected void write(XContentBuilder b, Object value) throws IOException {
+                    BytesRef ref = (BytesRef) value;
+                    b.utf8Value(ref.bytes, ref.offset, ref.length);
+                }
+            };
         }
 
         private static final TypeParser PARSER = new FixedTypeParser(c -> new TestMetadataMapper());

server/src/main/java/org/elasticsearch/index/IndexMode.java

@@ -26,6 +26,7 @@ import org.elasticsearch.index.mapper.MetadataFieldMapper;
 import org.elasticsearch.index.mapper.NestedLookup;
 import org.elasticsearch.index.mapper.ProvidedIdFieldMapper;
 import org.elasticsearch.index.mapper.RoutingFieldMapper;
+import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper;
 import org.elasticsearch.index.mapper.TsidExtractingIdFieldMapper;
 
@@ -110,6 +111,9 @@ public enum IndexMode {
         public boolean shouldValidateTimestamp() {
             return false;
         }
+
+        @Override
+        public void validateSourceFieldMapper(SourceFieldMapper sourceFieldMapper) {}
     },
     TIME_SERIES("time_series") {
         @Override
@@ -196,6 +200,13 @@ public enum IndexMode {
         public boolean shouldValidateTimestamp() {
             return true;
         }
+
+        @Override
+        public void validateSourceFieldMapper(SourceFieldMapper sourceFieldMapper) {
+            if (sourceFieldMapper.isSynthetic() == false) {
+                throw new IllegalArgumentException("time series indices only support synthetic source");
+            }
+        }
     };
 
     protected static String tsdbMode() {
@@ -310,6 +321,11 @@ public enum IndexMode {
      */
     public abstract boolean shouldValidateTimestamp();
 
+    /**
+     * Validates the source field mapper
+     */
+    public abstract void validateSourceFieldMapper(SourceFieldMapper sourceFieldMapper);
+
     /**
      * Parse a string into an {@link IndexMode}.
      */

server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java

@@ -93,13 +93,12 @@ public class DocumentMapper {
             }
         }
 
+        settings.getMode().validateMapping(mappingLookup);
         /*
          * Build an empty source loader to validate that the mapping is compatible
          * with the source loading strategy declared on the source field mapper.
          */
         sourceMapper().newSourceLoader(mapping());
-
-        settings.getMode().validateMapping(mappingLookup);
         if (settings.getIndexSortConfig().hasIndexSort() && mappers().nestedLookup() != NestedLookup.EMPTY) {
             throw new IllegalArgumentException("cannot have nested fields when index sort is activated");
         }

server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java

@@ -19,6 +19,7 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.query.QueryShardException;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.search.lookup.Source;
@@ -48,7 +49,16 @@ public class SourceFieldMapper extends MetadataFieldMapper {
         null,
         Explicit.IMPLICIT_TRUE,
         Strings.EMPTY_ARRAY,
-        Strings.EMPTY_ARRAY
+        Strings.EMPTY_ARRAY,
+        null
+    );
+
+    private static final SourceFieldMapper TSDB_DEFAULT = new SourceFieldMapper(
+        Mode.SYNTHETIC,
+        Explicit.IMPLICIT_TRUE,
+        Strings.EMPTY_ARRAY,
+        Strings.EMPTY_ARRAY,
+        IndexMode.TIME_SERIES
     );
 
     public static class Defaults {
@@ -79,7 +89,7 @@ public class SourceFieldMapper extends MetadataFieldMapper {
         private final Parameter<Mode> mode = new Parameter<>(
             "mode",
             true,
-            () -> null,
+            () -> getIndexMode() == IndexMode.TIME_SERIES ? Mode.SYNTHETIC : null,
             (n, c, o) -> Mode.valueOf(o.toString().toUpperCase(Locale.ROOT)),
             m -> toType(m).enabled.explicit() ? null : toType(m).mode,
             (b, n, v) -> b.field(n, v.toString().toLowerCase(Locale.ROOT)),
@@ -97,8 +107,11 @@ public class SourceFieldMapper extends MetadataFieldMapper {
             m -> Arrays.asList(toType(m).excludes)
         );
 
-        public Builder() {
+        private final IndexMode indexMode;
+
+        public Builder(IndexMode indexMode) {
             super(Defaults.NAME);
+            this.indexMode = indexMode;
         }
 
         public Builder setSynthetic() {
@@ -127,18 +140,30 @@ public class SourceFieldMapper extends MetadataFieldMapper {
                 throw new MapperParsingException("Cannot set both [mode] and [enabled] parameters");
             }
             if (isDefault()) {
-                return DEFAULT;
+                return indexMode == IndexMode.TIME_SERIES ? TSDB_DEFAULT : DEFAULT;
             }
-            return new SourceFieldMapper(
+            SourceFieldMapper sourceFieldMapper = new SourceFieldMapper(
                 mode.get(),
                 enabled.get(),
                 includes.getValue().toArray(String[]::new),
-                excludes.getValue().toArray(String[]::new)
+                excludes.getValue().toArray(String[]::new),
+                indexMode
             );
+            if (indexMode != null) {
+                indexMode.validateSourceFieldMapper(sourceFieldMapper);
+            }
+            return sourceFieldMapper;
+        }
+
+        private IndexMode getIndexMode() {
+            return indexMode;
         }
     }
 
-    public static final TypeParser PARSER = new ConfigurableTypeParser(c -> DEFAULT, c -> new Builder());
+    public static final TypeParser PARSER = new ConfigurableTypeParser(
+        c -> c.getIndexSettings().getMode() == IndexMode.TIME_SERIES ? TSDB_DEFAULT : DEFAULT,
+        c -> new Builder(c.getIndexSettings().getMode())
+    );
 
     static final class SourceFieldType extends MappedFieldType {
 
@@ -178,7 +203,9 @@ public class SourceFieldMapper extends MetadataFieldMapper {
     private final String[] excludes;
     private final SourceFilter sourceFilter;
 
-    private SourceFieldMapper(Mode mode, Explicit<Boolean> enabled, String[] includes, String[] excludes) {
+    private final IndexMode indexMode;
+
+    private SourceFieldMapper(Mode mode, Explicit<Boolean> enabled, String[] includes, String[] excludes, IndexMode indexMode) {
         super(new SourceFieldType((enabled.explicit() && enabled.value()) || (enabled.explicit() == false && mode != Mode.DISABLED)));
         assert enabled.explicit() == false || mode == null;
         this.mode = mode;
@@ -190,6 +217,7 @@ public class SourceFieldMapper extends MetadataFieldMapper {
             throw new IllegalArgumentException("filtering the stored _source is incompatible with synthetic source");
         }
         this.complete = stored() && sourceFilter == null;
+        this.indexMode = indexMode;
     }
 
     private static SourceFilter buildSourceFilter(String[] includes, String[] excludes) {
@@ -259,13 +287,13 @@ public class SourceFieldMapper extends MetadataFieldMapper {
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new Builder().init(this);
+        return new Builder(indexMode).init(this);
     }
 
     /**
      * Build something to load source {@code _source}.
      */
-    public <T> SourceLoader newSourceLoader(Mapping mapping) {
+    public SourceLoader newSourceLoader(Mapping mapping) {
         if (mode == Mode.SYNTHETIC) {
             return new SourceLoader.Synthetic(mapping);
         }

server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java

@@ -2571,7 +2571,13 @@ public class DocumentParserTests extends MapperServiceTestCase {
 
             @Override
             public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
-                throw new UnsupportedOperationException();
+                return new StringStoredFieldFieldLoader(name(), simpleName(), null) {
+                    @Override
+                    protected void write(XContentBuilder b, Object value) throws IOException {
+                        BytesRef ref = (BytesRef) value;
+                        b.utf8Value(ref.bytes, ref.offset, ref.length);
+                    }
+                };
             }
 
             private static final TypeParser PARSER = new FixedTypeParser(c -> new MockMetadataMapper());

server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java

@@ -398,7 +398,7 @@ public class SearchExecutionContextTests extends ESTestCase {
     public void testSyntheticSourceScriptLoading() throws IOException {
 
         // Build a mapping using synthetic source
-        SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder().setSynthetic().build();
+        SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null).setSynthetic().build();
         RootObjectMapper root = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).build(MapperBuilderContext.root(true));
         Mapping mapping = new Mapping(root, new MetadataFieldMapper[] { sourceMapper }, Map.of());
         MappingLookup lookup = MappingLookup.fromMapping(mapping);

x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java

@@ -31,6 +31,8 @@ import org.elasticsearch.index.mapper.MapperBuilderContext;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MappingLookup;
 import org.elasticsearch.index.mapper.SimpleMappedFieldType;
+import org.elasticsearch.index.mapper.SortedNumericDocValuesSyntheticFieldLoader;
+import org.elasticsearch.index.mapper.SourceLoader;
 import org.elasticsearch.index.mapper.SourceValueFetcher;
 import org.elasticsearch.index.mapper.TextSearchInfo;
 import org.elasticsearch.index.mapper.TimeSeriesParams;
@@ -707,4 +709,29 @@ public class UnsignedLongFieldMapper extends FieldMapper {
             );
         }
     }
+
+    @Override
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+        if (hasDocValues == false) {
+            throw new IllegalArgumentException(
+                "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it doesn't have doc values"
+            );
+        }
+        if (ignoreMalformed.value()) {
+            throw new IllegalArgumentException(
+                "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it ignores malformed numbers"
+            );
+        }
+        if (copyTo.copyToFields().isEmpty() != true) {
+            throw new IllegalArgumentException(
+                "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to"
+            );
+        }
+        return new SortedNumericDocValuesSyntheticFieldLoader(name(), simpleName(), ignoreMalformed()) {
+            @Override
+            protected void writeValue(XContentBuilder b, long value) throws IOException {
+                b.value(DocValueFormat.UNSIGNED_LONG_SHIFTED.format(value));
+            }
+        };
+    }
 }

x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java

@@ -11,6 +11,7 @@ import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.DocumentMapper;
@@ -32,6 +33,7 @@ import java.util.List;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.matchesPattern;
 
 public class UnsignedLongFieldMapperTests extends MapperTestCase {
 
@@ -352,11 +354,74 @@ public class UnsignedLongFieldMapperTests extends MapperTestCase {
 
     @Override
     protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
-        throw new AssumptionViolatedException("not supported");
+        assumeFalse("unsigned_long doesn't support ignore_malformed with synthetic _source", ignoreMalformed);
+        return new NumberSyntheticSourceSupport();
     }
 
     @Override
     protected IngestScriptSupport ingestScriptSupport() {
         throw new AssumptionViolatedException("not supported");
     }
+
+    final class NumberSyntheticSourceSupport implements SyntheticSourceSupport {
+        private final BigInteger nullValue = usually() ? null : BigInteger.valueOf(randomNonNegativeLong());
+
+        @Override
+        public SyntheticSourceExample example(int maxVals) {
+            if (randomBoolean()) {
+                Tuple<Object, Object> v = generateValue();
+                return new SyntheticSourceExample(v.v1(), v.v2(), this::mapping);
+            }
+            List<Tuple<Object, Object>> values = randomList(1, maxVals, this::generateValue);
+            List<Object> in = values.stream().map(Tuple::v1).toList();
+            List<Object> outList = values.stream().map(Tuple::v2).sorted().toList();
+            Object out = outList.size() == 1 ? outList.get(0) : outList;
+            return new SyntheticSourceExample(in, out, this::mapping);
+        }
+
+        private Tuple<Object, Object> generateValue() {
+            if (nullValue != null && randomBoolean()) {
+                return Tuple.tuple(null, nullValue);
+            }
+            long n = randomNonNegativeLong();
+            BigInteger b = BigInteger.valueOf(n);
+            if (b.signum() < 0) {
+                b = b.add(BigInteger.ONE.shiftLeft(64));
+            }
+            return Tuple.tuple(n, b);
+        }
+
+        private void mapping(XContentBuilder b) throws IOException {
+            minimalMapping(b);
+            if (nullValue != null) {
+                b.field("null_value", nullValue);
+            }
+            if (rarely()) {
+                b.field("index", false);
+            }
+            if (rarely()) {
+                b.field("store", false);
+            }
+        }
+
+        @Override
+        public List<SyntheticSourceInvalidExample> invalidExample() {
+            return List.of(
+                new SyntheticSourceInvalidExample(
+                    matchesPattern("field \\[field] of type \\[.+] doesn't support synthetic source because it doesn't have doc values"),
+                    b -> {
+                        minimalMapping(b);
+                        b.field("doc_values", false);
+                    }
+                ),
+                new SyntheticSourceInvalidExample(
+                    matchesPattern("field \\[field] of type \\[.+] doesn't support synthetic source because it ignores malformed numbers"),
+                    b -> {
+                        minimalMapping(b);
+                        b.field("ignore_malformed", true);
+                    }
+                )
+            );
+        }
+    }
 }

x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/70_time_series.yml

@@ -71,6 +71,32 @@ fetch the _tsid:
   - match: {hits.hits.1.fields.metricset: [aa]}
   - match: {hits.hits.1.fields.ul: [9223372036854775808]}
 
+---
+fetch the _source:
+  - skip:
+      version: " - 8.6.99"
+      reason: synthetic source support to unsigned long added in 8.7.0
+
+  - do:
+      search:
+        index: test
+        body:
+          fields: [_tsid, metricset, ul]
+          query:
+            query_string:
+              query: '+@timestamp:"2021-04-28T18:35:24.467Z" +metricset:aa'
+          sort:   [ _tsid ]
+
+  - match: {hits.total.value: 2}
+  - match: {hits.hits.0._source.voltage: 7.2}
+  - match: {hits.hits.0._source.metricset: aa}
+  - match: {hits.hits.0._source.@timestamp: 2021-04-28T18:35:24.467Z}
+  - match: {hits.hits.0._source.ul: 9223372036854775807}
+  - match: {hits.hits.1._source.voltage: 3.2}
+  - match: {hits.hits.1._source.metricset: aa}
+  - match: {hits.hits.1._source.@timestamp: 2021-04-28T18:35:24.467Z}
+  - match: {hits.hits.1._source.ul: 9223372036854775808}
+
 ---
 # Sort order is of unsigned_long fields is not the one we would expect.
 # This is caused by the encoding of unsigned_long as a signed long before

x-pack/plugin/rollup/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/rollup/10_basic.yml

@@ -90,7 +90,7 @@ setup:
 "Downsample index":
   - skip:
       version: " - 8.4.99"
-      reason: "rollup renamed to downsample in 8.5.0"
+      reason: "Downsampling GA-ed in 8.7.0"
 
   - do:
       indices.downsample:
@@ -110,23 +110,23 @@ setup:
 
   - length: { hits.hits: 4 }
   - match:  { hits.hits.0._source._doc_count: 2 }
-  - match:  { hits.hits.0._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
+  - match:  { hits.hits.0._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
   - match:  { hits.hits.0._source.metricset: pod }
   - match:  { hits.hits.0._source.@timestamp: 2021-04-28T18:00:00.000Z }
-  - match:  { hits.hits.0._source.k8s\.pod\.multi-counter: 21 }
-  - match:  { hits.hits.0._source.k8s\.pod\.multi-gauge.min: 90 }
-  - match:  { hits.hits.0._source.k8s\.pod\.multi-gauge.max: 200 }
-  - match:  { hits.hits.0._source.k8s\.pod\.multi-gauge.sum: 726 }
-  - match:  { hits.hits.0._source.k8s\.pod\.multi-gauge.value_count: 6 }
-  - match:  { hits.hits.0._source.k8s\.pod\.network\.tx.min: 2001818691 }
-  - match:  { hits.hits.0._source.k8s\.pod\.network\.tx.max: 2005177954 }
-  - match:  { hits.hits.0._source.k8s\.pod\.network\.tx.value_count: 2 }
-  - match:  { hits.hits.0._source.k8s\.pod\.ip: "10.10.55.26" }
-  - match:  { hits.hits.0._source.k8s\.pod\.created_at: "2021-04-28T19:35:00.000Z" }
-  - match:  { hits.hits.0._source.k8s\.pod\.number_of_containers: 2 }
-  - match:  { hits.hits.0._source.k8s\.pod\.tags: ["backend", "prod", "us-west1"] }
-  - match:  { hits.hits.0._source.k8s\.pod\.values: [1, 1, 3] }
-  - is_true: hits.hits.0._source.k8s\.pod\.running
+  - match:  { hits.hits.0._source.k8s.pod.multi-counter: 21 }
+  - match:  { hits.hits.0._source.k8s.pod.multi-gauge.min: 90 }
+  - match:  { hits.hits.0._source.k8s.pod.multi-gauge.max: 200 }
+  - match:  { hits.hits.0._source.k8s.pod.multi-gauge.sum: 726 }
+  - match:  { hits.hits.0._source.k8s.pod.multi-gauge.value_count: 6 }
+  - match:  { hits.hits.0._source.k8s.pod.network.tx.min: 2001818691 }
+  - match:  { hits.hits.0._source.k8s.pod.network.tx.max: 2005177954 }
+  - match:  { hits.hits.0._source.k8s.pod.network.tx.value_count: 2 }
+  - match:  { hits.hits.0._source.k8s.pod.ip: "10.10.55.26" }
+  - match:  { hits.hits.0._source.k8s.pod.created_at: "2021-04-28T19:35:00.000Z" }
+  - match:  { hits.hits.0._source.k8s.pod.number_of_containers: 2 }
+  - match:  { hits.hits.0._source.k8s.pod.tags: ["backend", "prod", "us-west1"] }
+  - match:  { hits.hits.0._source.k8s.pod.values: [1, 1, 3] }
+  - is_true: hits.hits.0._source.k8s.pod.running
 
   # Assert rollup index settings
   - do:
@@ -345,8 +345,8 @@ setup:
 ---
 "Downsample a downsampled index":
   - skip:
-      version: " - 8.4.99"
-      reason: "Rollup of rollups introduced in 8.5.0"
+      version: " - 8.6.99"
+      reason: "Rollup GA-ed in 8.7.0"
 
   - do:
       indices.downsample:
@@ -398,29 +398,29 @@ setup:
 
   - length: { hits.hits: 3 }
   - match: { hits.hits.0._source._doc_count: 2 }
-  - match: { hits.hits.0._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
+  - match: { hits.hits.0._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
   - match: { hits.hits.0._source.metricset: pod }
   - match: { hits.hits.0._source.@timestamp: 2021-04-28T18:00:00.000Z }
-  - match: { hits.hits.0._source.k8s\.pod\.multi-counter: 21 }
-  - match: { hits.hits.0._source.k8s\.pod\.multi-gauge.min: 90 }
-  - match: { hits.hits.0._source.k8s\.pod\.multi-gauge.max: 200 }
-  - match: { hits.hits.0._source.k8s\.pod\.multi-gauge.sum: 726 }
-  - match: { hits.hits.0._source.k8s\.pod\.multi-gauge.value_count: 6 }
-  - match: { hits.hits.0._source.k8s\.pod\.network\.tx.min: 2001818691 }
-  - match: { hits.hits.0._source.k8s\.pod\.network\.tx.max: 2005177954 }
-  - match: { hits.hits.0._source.k8s\.pod\.network\.tx.value_count: 2 }
-  - match: { hits.hits.0._source.k8s\.pod\.ip: "10.10.55.26" }
-  - match: { hits.hits.0._source.k8s\.pod\.created_at: "2021-04-28T19:35:00.000Z" }
-  - match: { hits.hits.0._source.k8s\.pod\.number_of_containers: 2 }
-  - match: { hits.hits.0._source.k8s\.pod\.tags: [ "backend", "prod", "us-west1" ] }
-  - match: { hits.hits.0._source.k8s\.pod\.values: [ 1, 1, 3 ] }
-
-  - match: { hits.hits.1._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
+  - match: { hits.hits.0._source.k8s.pod.multi-counter: 21 }
+  - match: { hits.hits.0._source.k8s.pod.multi-gauge.min: 90 }
+  - match: { hits.hits.0._source.k8s.pod.multi-gauge.max: 200 }
+  - match: { hits.hits.0._source.k8s.pod.multi-gauge.sum: 726 }
+  - match: { hits.hits.0._source.k8s.pod.multi-gauge.value_count: 6 }
+  - match: { hits.hits.0._source.k8s.pod.network.tx.min: 2001818691 }
+  - match: { hits.hits.0._source.k8s.pod.network.tx.max: 2005177954 }
+  - match: { hits.hits.0._source.k8s.pod.network.tx.value_count: 2 }
+  - match: { hits.hits.0._source.k8s.pod.ip: "10.10.55.26" }
+  - match: { hits.hits.0._source.k8s.pod.created_at: "2021-04-28T19:35:00.000Z" }
+  - match: { hits.hits.0._source.k8s.pod.number_of_containers: 2 }
+  - match: { hits.hits.0._source.k8s.pod.tags: [ "backend", "prod", "us-west1" ] }
+  - match: { hits.hits.0._source.k8s.pod.values: [ 1, 1, 3 ] }
+
+  - match: { hits.hits.1._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
   - match: { hits.hits.1._source.metricset: pod }
   - match: { hits.hits.1._source.@timestamp: 2021-04-28T20:00:00.000Z }
   - match: { hits.hits.1._source._doc_count: 2 }
 
-  - match: { hits.hits.2._source.k8s\.pod\.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
+  - match: { hits.hits.2._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
   - match: { hits.hits.2._source.metricset: pod }
   - match: { hits.hits.2._source.@timestamp: 2021-04-28T18:00:00.000Z }
   - match: { hits.hits.2._source._doc_count: 4 }

x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java

@@ -214,7 +214,7 @@ public class DownsampleActionSingleNodeTests extends ESSingleNodeTestCase {
         mapping.startObject(FIELD_LABEL_DOUBLE).field("type", "double").endObject();
         mapping.startObject(FIELD_LABEL_INTEGER).field("type", "integer").endObject();
         mapping.startObject(FIELD_LABEL_KEYWORD).field("type", "keyword").endObject();
-        mapping.startObject(FIELD_LABEL_TEXT).field("type", "text").endObject();
+        mapping.startObject(FIELD_LABEL_TEXT).field("type", "text").field("store", "true").endObject();
         mapping.startObject(FIELD_LABEL_BOOLEAN).field("type", "boolean").endObject();
         mapping.startObject(FIELD_LABEL_IPv4_ADDRESS).field("type", "ip").endObject();
         mapping.startObject(FIELD_LABEL_IPv6_ADDRESS).field("type", "ip").endObject();