Composable templates: add a default mapping for @timestamp (#59244)

This adds a low precendece mapping for the `@timestamp` field with
type `date`.
This will aid with the bootstrapping of data streams as a timestamp
mapping can be omitted when nanos precision is not needed.

build.gradle

@@ -174,8 +174,8 @@ tasks.register("verifyVersions") {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = true
-final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
+boolean bwc_tests_enabled = false
+final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/59244" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

docs/reference/data-streams/set-up-a-data-stream.asciidoc

@@ -96,7 +96,7 @@ PUT /_ilm/policy/logs_policy
 Each data stream requires an <<indices-templates,index template>>. The data
 stream uses this template to create its backing indices.
 
-Index templates for data streams must contain:
+An index template for a data stream must contain:
 
 * A name or wildcard (`*`) pattern for the data stream in the `index_patterns`
 property.
@@ -135,16 +135,22 @@ this pattern.
 property. The `@timestamp` field must be included in every document indexed to
 the data stream.
 
-* A <<date,`date`>> or <<date_nanos,`date_nanos`>> field mapping for the
-`@timestamp` field.
+The template can also contain:
+
+* An optional field mapping for the `@timestamp` field. Both the <<date,`date`>> and
+<<date_nanos,`date_nanos`>> field data types are supported. If no mapping is specified,
+a <<date,`date`>> field data type with default options is used.
++
+This mapping can include other <<mapping-params,mapping parameters>>, such as
+<<mapping-date-format,`format`>>.
 +
 IMPORTANT: Carefully consider the `@timestamp` field's mapping, including
-<<mapping-params,mapping parameters>> such as <<mapping-date-format,`format`>>.
+its <<mapping-params,mapping parameters>>.
 Once the stream is created, you can only update the `@timestamp` field's mapping
 by reindexing the data stream. See
 <<data-streams-use-reindex-to-change-mappings-settings>>.
 
-* If you intend to use {ilm-init}, you must specify the
+* If you intend to use {ilm-init}, the
   <<configure-a-data-stream-ilm-policy,lifecycle policy>> in the
   `index.lifecycle.name` setting.
 
@@ -171,13 +177,6 @@ PUT /_index_template/logs_data_stream
     "timestamp_field": "@timestamp"
   },
   "template": {
-    "mappings": {
-      "properties": {
-        "@timestamp": {
-          "type": "date"
-        }
-      }
-    },
     "settings": {
       "index.lifecycle.name": "logs_policy"
     }

docs/reference/data-streams/use-a-data-stream.asciidoc

@@ -21,15 +21,6 @@ PUT /_index_template/logs_data_stream
   "index_patterns": [ "logs*" ],
   "data_stream": {
     "timestamp_field": "@timestamp"
-  },
-  "template": {
-    "mappings": {
-      "properties": {
-        "@timestamp": {
-          "type": "date"
-        }
-      }
-    }
   }
 }
 

rest-api-spec/src/main/resources/rest-api-spec/test/indices.resolve_index/10_basic_resolve_index.yml

@@ -35,8 +35,8 @@ setup:
 ---
 "Resolve index with indices and aliases":
   - skip:
-      version: " - 7.8.99"
-      reason: "resolve index api only supported in 7.9+"
+      version: " - 7.9.99"
+      reason: "enable in 7.9+ when backported"
 
   - do:
       indices.resolve_index:
@@ -63,8 +63,8 @@ setup:
 ---
 "Resolve index with hidden and closed indices":
   - skip:
-      version: " - 7.8.99"
-      reason: change after backporting
+      version: " - 7.9.99"
+      reason: "enable in 7.9+ when backported"
 
   - do:
       indices.resolve_index:

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java

@@ -515,7 +515,7 @@ public class MetadataCreateIndexService {
         List<Map<String, Object>> result = new ArrayList<>();
 
         List<CompressedXContent> templateMappings =
-            MetadataIndexTemplateService.collectMappings(currentState, templateName, indexName);
+            MetadataIndexTemplateService.collectMappings(currentState, templateName, indexName, xContentRegistry);
         for (CompressedXContent templateMapping : templateMappings) {
             Map<String, Object> parsedTemplateMapping = MapperService.parseMapping(xContentRegistry, templateMapping.string());
             result.add(parsedTemplateMapping);

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java

@@ -68,6 +68,7 @@ import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -87,6 +88,14 @@ import static org.elasticsearch.indices.cluster.IndicesClusterStateService.Alloc
  */
 public class MetadataIndexTemplateService {
 
+    public static final String DEFAULT_TIMESTAMP_FIELD = "@timestamp";
+    public static final String DEFAULT_TIMESTAMP_MAPPING = "{\n" +
+        "      \"properties\": {\n" +
+        "        \"@timestamp\": {\n" +
+        "          \"type\": \"date\"\n" +
+        "        }\n" +
+        "      }\n" +
+        "    }";
     private static final Logger logger = LogManager.getLogger(MetadataIndexTemplateService.class);
     private final ClusterService clusterService;
     private final AliasValidator aliasValidator;
@@ -940,7 +949,8 @@ public class MetadataIndexTemplateService {
      */
     public static List<CompressedXContent> collectMappings(final ClusterState state,
                                                            final String templateName,
-                                                           final String indexName) {
+                                                           final String indexName,
+                                                           final NamedXContentRegistry xContentRegistry) throws Exception {
         final ComposableIndexTemplate template = state.metadata().templatesV2().get(templateName);
         assert template != null : "attempted to resolve mappings for a template [" + templateName +
             "] that did not exist in the cluster state";
@@ -955,11 +965,16 @@ public class MetadataIndexTemplateService {
             .map(ComponentTemplate::template)
             .map(Template::mappings)
             .filter(Objects::nonNull)
-            .collect(Collectors.toList());
+            .collect(Collectors.toCollection(LinkedList::new));
         // Add the actual index template's mappings, since it takes the highest precedence
         Optional.ofNullable(template.template())
             .map(Template::mappings)
             .ifPresent(mappings::add);
+        if (template.getDataStreamTemplate() != null && indexName.startsWith(DataStream.BACKING_INDEX_PREFIX)) {
+            // add a default mapping for the `@timestamp` field, at the lowest precedence, to make bootstrapping data streams more
+            // straightforward as all backing indices are required to have a timestamp field
+            mappings.add(0, new CompressedXContent(wrapMappingsIfNecessary(DEFAULT_TIMESTAMP_MAPPING, xContentRegistry)));
+        }
 
         // Only include _timestamp mapping snippet if creating backing index.
         if (indexName.startsWith(DataStream.BACKING_INDEX_PREFIX)) {
@@ -1136,7 +1151,7 @@ public class MetadataIndexTemplateService {
                     }
                 }
 
-                List<CompressedXContent> mappings = collectMappings(stateWithIndex, templateName, indexName );
+                List<CompressedXContent> mappings = collectMappings(stateWithIndex, templateName, indexName, xContentRegistry);
                 try {
                     MapperService mapperService = tempIndexService.mapperService();
                     for (CompressedXContent mapping : mappings) {

server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java

@@ -74,6 +74,7 @@ import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 
 import static java.util.Collections.singletonList;
+import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.DEFAULT_TIMESTAMP_FIELD;
 import static org.elasticsearch.common.settings.Settings.builder;
 import static org.elasticsearch.indices.ShardLimitValidatorTests.createTestShardLimitService;
 import static org.hamcrest.CoreMatchers.containsString;
@@ -726,7 +727,8 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
             List.of("ct_low", "ct_high"), 0L, 1L, null, null);
         state = service.addIndexTemplateV2(state, true, "my-template", it);
 
-        List<CompressedXContent> mappings = MetadataIndexTemplateService.collectMappings(state, "my-template", "my-index");
+        List<CompressedXContent> mappings = MetadataIndexTemplateService.collectMappings(state, "my-template", "my-index",
+            xContentRegistry());
 
         assertNotNull(mappings);
         assertThat(mappings.size(), equalTo(3));
@@ -789,7 +791,8 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
             List.of("ct_low", "ct_high"), 0L, 1L, null, null);
         state = service.addIndexTemplateV2(state, true, "my-template", it);
 
-        List<CompressedXContent> mappings = MetadataIndexTemplateService.collectMappings(state, "my-template", "my-index");
+        List<CompressedXContent> mappings = MetadataIndexTemplateService.collectMappings(state, "my-template", "my-index",
+            xContentRegistry());
 
         assertNotNull(mappings);
         assertThat(mappings.size(), equalTo(3));
@@ -804,7 +807,6 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
                 }
             })
             .collect(Collectors.toList());
-
         assertThat(parsedMappings.get(0),
             equalTo(Map.of("_doc", Map.of("properties", Map.of("field2", Map.of("type", "text"))))));
         assertThat(parsedMappings.get(1),
@@ -813,6 +815,199 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
             equalTo(Map.of("_doc", Map.of("properties", Map.of("field3", Map.of("type", "integer"))))));
     }
 
+    public void testDefinedTimestampMappingIsAddedForDataStreamTemplates() throws Exception {
+        final MetadataIndexTemplateService service = getMetadataIndexTemplateService();
+        ClusterState state = ClusterState.EMPTY_STATE;
+
+        ComponentTemplate ct1 = new ComponentTemplate(new Template(null,
+            new CompressedXContent("{\n" +
+                "      \"properties\": {\n" +
+                "        \"field1\": {\n" +
+                "          \"type\": \"keyword\"\n" +
+                "        }\n" +
+                "      }\n" +
+                "    }"), null), null, null);
+
+        state = service.addComponentTemplate(state, true, "ct1", ct1);
+
+        {
+            ComposableIndexTemplate it = new ComposableIndexTemplate(List.of("logs*"),
+                new Template(null,
+                    new CompressedXContent("{\n" +
+                        "    \"properties\": {\n" +
+                        "      \"field2\": {\n" +
+                        "        \"type\": \"integer\"\n" +
+                        "      }\n" +
+                        "    }\n" +
+                        "  }"), null),
+                List.of("ct1"), 0L, 1L, null, new ComposableIndexTemplate.DataStreamTemplate(DEFAULT_TIMESTAMP_FIELD));
+            state = service.addIndexTemplateV2(state, true, "logs-data-stream-template", it);
+
+            List<CompressedXContent> mappings = MetadataIndexTemplateService.collectMappings(state, "logs-data-stream-template",
+                DataStream.getDefaultBackingIndexName("logs", 1L), xContentRegistry());
+
+            assertNotNull(mappings);
+            assertThat(mappings.size(), equalTo(4));
+            List<Map<String, Object>> parsedMappings = mappings.stream()
+                .map(m -> {
+                    try {
+                        return MapperService.parseMapping(new NamedXContentRegistry(List.of()), m.string());
+                    } catch (Exception e) {
+                        logger.error(e);
+                        fail("failed to parse mappings: " + m.string());
+                        return null;
+                    }
+                })
+                .collect(Collectors.toList());
+
+            assertThat(parsedMappings.get(0),
+                equalTo(Map.of("_doc", Map.of("properties", Map.of(DEFAULT_TIMESTAMP_FIELD, Map.of("type", "date"))))));
+            assertThat(parsedMappings.get(1),
+                equalTo(Map.of("_doc", Map.of("properties", Map.of("field1", Map.of("type", "keyword"))))));
+            assertThat(parsedMappings.get(2),
+                equalTo(Map.of("_doc", Map.of("properties", Map.of("field2", Map.of("type", "integer"))))));
+        }
+
+        {
+            // indices matched by templates without the data stream field defined don't get the default @timestamp mapping
+            ComposableIndexTemplate it = new ComposableIndexTemplate(List.of("timeseries*"),
+                new Template(null,
+                    new CompressedXContent("{\n" +
+                        "    \"properties\": {\n" +
+                        "      \"field2\": {\n" +
+                        "        \"type\": \"integer\"\n" +
+                        "      }\n" +
+                        "    }\n" +
+                        "  }"), null),
+                List.of("ct1"), 0L, 1L, null, null);
+            state = service.addIndexTemplateV2(state, true, "timeseries-template", it);
+
+            List<CompressedXContent> mappings = MetadataIndexTemplateService.collectMappings(state, "timeseries-template", "timeseries",
+                xContentRegistry());
+
+            assertNotNull(mappings);
+            assertThat(mappings.size(), equalTo(2));
+            List<Map<String, Object>> parsedMappings = mappings.stream()
+                .map(m -> {
+                    try {
+                        return MapperService.parseMapping(new NamedXContentRegistry(List.of()), m.string());
+                    } catch (Exception e) {
+                        logger.error(e);
+                        fail("failed to parse mappings: " + m.string());
+                        return null;
+                    }
+                })
+                .collect(Collectors.toList());
+
+            assertThat(parsedMappings.get(0),
+                equalTo(Map.of("_doc", Map.of("properties", Map.of("field1", Map.of("type", "keyword"))))));
+            assertThat(parsedMappings.get(1),
+                equalTo(Map.of("_doc", Map.of("properties", Map.of("field2", Map.of("type", "integer"))))));
+
+            // a default @timestamp mapping will not be added if the matching template doesn't have the data stream field configured, even
+            // if the index name matches that of a data stream backing index
+            mappings = MetadataIndexTemplateService.collectMappings(state, "timeseries-template",
+                DataStream.getDefaultBackingIndexName("timeseries", 1L), xContentRegistry());
+
+            assertNotNull(mappings);
+            assertThat(mappings.size(), equalTo(2));
+            parsedMappings = mappings.stream()
+                .map(m -> {
+                    try {
+                        return MapperService.parseMapping(new NamedXContentRegistry(List.of()), m.string());
+                    } catch (Exception e) {
+                        logger.error(e);
+                        fail("failed to parse mappings: " + m.string());
+                        return null;
+                    }
+                })
+                .collect(Collectors.toList());
+
+            assertThat(parsedMappings.get(0),
+                equalTo(Map.of("_doc", Map.of("properties", Map.of("field1", Map.of("type", "keyword"))))));
+            assertThat(parsedMappings.get(1),
+                equalTo(Map.of("_doc", Map.of("properties", Map.of("field2", Map.of("type", "integer"))))));
+        }
+    }
+
+    public void testUserDefinedMappingTakesPrecedenceOverDefault() throws Exception {
+        final MetadataIndexTemplateService service = getMetadataIndexTemplateService();
+        ClusterState state = ClusterState.EMPTY_STATE;
+
+        {
+            // user defines a @timestamp mapping as part of a component template
+            ComponentTemplate ct1 = new ComponentTemplate(new Template(null,
+                new CompressedXContent("{\n" +
+                    "      \"properties\": {\n" +
+                    "        \"@timestamp\": {\n" +
+                    "          \"type\": \"date_nanos\"\n" +
+                    "        }\n" +
+                    "      }\n" +
+                    "    }"), null), null, null);
+
+            state = service.addComponentTemplate(state, true, "ct1", ct1);
+            ComposableIndexTemplate it = new ComposableIndexTemplate(List.of("logs*"), null, List.of("ct1"), 0L, 1L, null,
+                new ComposableIndexTemplate.DataStreamTemplate(DEFAULT_TIMESTAMP_FIELD));
+            state = service.addIndexTemplateV2(state, true, "logs-template", it);
+
+            List<CompressedXContent> mappings = MetadataIndexTemplateService.collectMappings(state, "logs-template",
+                DataStream.getDefaultBackingIndexName("logs", 1L), xContentRegistry());
+
+            assertNotNull(mappings);
+            assertThat(mappings.size(), equalTo(3));
+            List<Map<String, Object>> parsedMappings = mappings.stream()
+                .map(m -> {
+                    try {
+                        return MapperService.parseMapping(new NamedXContentRegistry(List.of()), m.string());
+                    } catch (Exception e) {
+                        logger.error(e);
+                        fail("failed to parse mappings: " + m.string());
+                        return null;
+                    }
+                })
+                .collect(Collectors.toList());
+            assertThat(parsedMappings.get(0),
+                equalTo(Map.of("_doc", Map.of("properties", Map.of(DEFAULT_TIMESTAMP_FIELD, Map.of("type", "date"))))));
+            assertThat(parsedMappings.get(1),
+                equalTo(Map.of("_doc", Map.of("properties", Map.of(DEFAULT_TIMESTAMP_FIELD, Map.of("type", "date_nanos"))))));
+        }
+
+        {
+            // user defines a @timestamp mapping as part of a composable index template
+            Template template = new Template(null, new CompressedXContent("{\n" +
+                "      \"properties\": {\n" +
+                "        \"@timestamp\": {\n" +
+                "          \"type\": \"date_nanos\"\n" +
+                "        }\n" +
+                "      }\n" +
+                "    }"), null);
+            ComposableIndexTemplate it = new ComposableIndexTemplate(List.of("timeseries*"), template, null, 0L, 1L, null,
+                new ComposableIndexTemplate.DataStreamTemplate(DEFAULT_TIMESTAMP_FIELD));
+            state = service.addIndexTemplateV2(state, true, "timeseries-template", it);
+
+            List<CompressedXContent> mappings = MetadataIndexTemplateService.collectMappings(state, "timeseries-template",
+                DataStream.getDefaultBackingIndexName("timeseries-template", 1L), xContentRegistry());
+
+            assertNotNull(mappings);
+            assertThat(mappings.size(), equalTo(3));
+            List<Map<String, Object>> parsedMappings = mappings.stream()
+                .map(m -> {
+                    try {
+                        return MapperService.parseMapping(new NamedXContentRegistry(List.of()), m.string());
+                    } catch (Exception e) {
+                        logger.error(e);
+                        fail("failed to parse mappings: " + m.string());
+                        return null;
+                    }
+                })
+                .collect(Collectors.toList());
+            assertThat(parsedMappings.get(0),
+                equalTo(Map.of("_doc", Map.of("properties", Map.of(DEFAULT_TIMESTAMP_FIELD, Map.of("type", "date"))))));
+            assertThat(parsedMappings.get(1),
+                equalTo(Map.of("_doc", Map.of("properties", Map.of(DEFAULT_TIMESTAMP_FIELD, Map.of("type", "date_nanos"))))));
+        }
+    }
+
     public void testResolveSettings() throws Exception {
         final MetadataIndexTemplateService service = getMetadataIndexTemplateService();
         ClusterState state = ClusterState.EMPTY_STATE;
@@ -1100,16 +1295,8 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
                 .build())
             .build();
 
-        String mapping = "{\n" +
-            "      \"properties\": {\n" +
-            "        \"@timestamp\": {\n" +
-            "          \"type\": \"date\"\n" +
-            "        }\n" +
-            "      }\n" +
-            "    }";
-        Template mappingTemplate = new Template(null, new CompressedXContent(mapping), null);
         ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList("logs-*-*"),
-            mappingTemplate, null, 100L, null, null, new ComposableIndexTemplate.DataStreamTemplate("@timestamp"));
+            null, null, 100L, null, null, new ComposableIndexTemplate.DataStreamTemplate("@timestamp"));
 
         state = service.addIndexTemplateV2(state, false, "logs", template);
 
@@ -1152,7 +1339,7 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
 
         // Change the pattern to one that doesn't match the data stream
         e = expectThrows(IllegalArgumentException.class, () -> {
-            ComposableIndexTemplate newTemplate = new ComposableIndexTemplate(Collections.singletonList("logs-postgres-*"), mappingTemplate,
+            ComposableIndexTemplate newTemplate = new ComposableIndexTemplate(Collections.singletonList("logs-postgres-*"), null,
                 null, 100L, null, null, new ComposableIndexTemplate.DataStreamTemplate("@timestamp"));
             service.addIndexTemplateV2(stateWithDS, false, "logs", newTemplate);
         });
@@ -1162,7 +1349,7 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
                 "cause data streams [unreferenced, logs-mysql-default] to no longer match a data stream template"));
 
         // Add an additional template that matches our data stream at a lower priority
-        ComposableIndexTemplate mysqlTemplate = new ComposableIndexTemplate(Collections.singletonList("logs-mysql-*"), mappingTemplate,
+        ComposableIndexTemplate mysqlTemplate = new ComposableIndexTemplate(Collections.singletonList("logs-mysql-*"), null,
             null, 50L, null, null, new ComposableIndexTemplate.DataStreamTemplate("@timestamp"));
         ClusterState stateWithDSAndTemplate = service.addIndexTemplateV2(stateWithDS, false, "logs-mysql", mysqlTemplate);
 

x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/100_delete_by_query.yml

@@ -12,11 +12,6 @@
         name: my-template1
         body:
           index_patterns: [simple-data-stream1]
-          template:
-            mappings:
-              properties:
-                '@timestamp':
-                  type: date
           data_stream:
             timestamp_field: '@timestamp'
 

x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/10_basic.yml

@@ -1,6 +1,9 @@
 setup:
   - skip:
       features: allowed_warnings
+      version: " - 7.9.99"
+      reason: "enable in 7.9+ when backported"
+
   - do:
       allowed_warnings:
         - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation"
@@ -9,10 +12,6 @@ setup:
         body:
           index_patterns: [simple-data-stream1]
           template:
-            mappings:
-              properties:
-                '@timestamp':
-                  type: date
             settings:
               index.number_of_replicas: 0
           data_stream:
@@ -28,15 +27,15 @@ setup:
             mappings:
               properties:
                 '@timestamp':
-                  type: date
+                  type: date_nanos
           data_stream:
             timestamp_field: '@timestamp'
 
 ---
 "Create data stream":
   - skip:
-      version: " - 7.8.99"
-      reason: "data streams only supported in 7.9+"
+      version: " - 7.9.99"
+      reason: "enable in 7.9+ when backported"
 
   - do:
       indices.create_data_stream:
@@ -65,6 +64,18 @@ setup:
   - match: { data_streams.1.indices.0.index_name: '.ds-simple-data-stream2-000001' }
   - match: { data_streams.1.template: 'my-template2' }
 
+  - do:
+      indices.get_mapping:
+        index: .ds-simple-data-stream1-000001
+        expand_wildcards: hidden
+  - match: { \.ds-simple-data-stream1-000001.mappings.properties.@timestamp.type: 'date' }
+
+  - do:
+      indices.get_mapping:
+        index: .ds-simple-data-stream2-000001
+        expand_wildcards: hidden
+  - match: { \.ds-simple-data-stream2-000001.mappings.properties.@timestamp.type: 'date_nanos' }
+
   - do:
       index:
         index:  simple-data-stream1
@@ -97,8 +108,8 @@ setup:
 ---
 "Create data stream with invalid name":
   - skip:
-      version: " - 7.8.99"
-      reason: "data streams only supported in 7.9+"
+      version: " - 7.9.99"
+      reason: "enable in 7.9+ when backported"
 
   - do:
       catch: bad_request
@@ -112,7 +123,7 @@ setup:
 ---
 "Get data stream":
   - skip:
-      version: " - 7.8.99"
+      version: " - 7.9.99"
       reason: "change to 7.8.99 after backport"
 
   - do:
@@ -177,8 +188,8 @@ setup:
 ---
 "Delete data stream with backing indices":
   - skip:
-      version: " - 7.8.99"
-      reason: "data streams only supported in 7.9+"
+      version: " - 7.9.99"
+      reason: "enable in 7.9+ when backported"
 
   - do:
       indices.create_data_stream:
@@ -221,8 +232,8 @@ setup:
 ---
 "append-only writes to backing indices prohobited":
   - skip:
-      version: " - 7.8.99"
-      reason: "data streams only supported in 7.9+"
+      version: " - 7.9.99"
+      reason: "enable in 7.9+ when backported"
       features: allowed_warnings
 
   - do:
@@ -232,11 +243,6 @@ setup:
         name: generic_logs_template
         body:
           index_patterns: logs-*
-          template:
-            mappings:
-              properties:
-                '@timestamp':
-                  type: date
           data_stream:
             timestamp_field: '@timestamp'
 
@@ -293,11 +299,6 @@ setup:
         name: generic_logs_template
         body:
           index_patterns: logs-*
-          template:
-            mappings:
-              properties:
-                '@timestamp':
-                  type: date
           data_stream:
             timestamp_field: '@timestamp'
 

x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/110_update_by_query.yml

@@ -12,11 +12,6 @@
         name: my-template1
         body:
           index_patterns: [simple-data-stream1]
-          template:
-            mappings:
-              properties:
-                '@timestamp':
-                  type: date
           data_stream:
             timestamp_field: '@timestamp'
 

x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/20_unsupported_apis.yml

@@ -1,8 +1,8 @@
 ---
 "Test apis that do not supported data streams":
   - skip:
-      version: " - 7.8.99"
-      reason: "data streams only supported in 7.9+"
+      version: " - 7.9.99"
+      reason: "enable in 7.9+ when backported"
       features: allowed_warnings
 
   - do:
@@ -12,11 +12,6 @@
         name: my-template
         body:
           index_patterns: [logs-*]
-          template:
-            mappings:
-              properties:
-                '@timestamp':
-                  type: date
           data_stream:
             timestamp_field: '@timestamp'
 
@@ -55,8 +50,8 @@
 ---
 "Prohibit clone on data stream's write index":
   - skip:
-      version: " - 7.8.99"
-      reason: "data streams only supported in 7.9+"
+      version: " - 7.9.99"
+      reason: "enable in 7.9+ when backported"
       features: allowed_warnings
 
   - do:
@@ -66,11 +61,6 @@
         name: my-template1
         body:
           index_patterns: [simple-data-stream1]
-          template:
-            mappings:
-              properties:
-                '@timestamp':
-                  type: date
           data_stream:
             timestamp_field: '@timestamp'
 
@@ -110,7 +100,7 @@
 ---
 "Prohibit shrink on data stream's write index":
   - skip:
-      version: " - 7.8.99"
+      version: " - 7.9.99"
       reason: "data streams only supported in 7.9+"
       features: allowed_warnings
 
@@ -121,11 +111,6 @@
         name: my-template1
         body:
           index_patterns: [simple-data-stream1]
-          template:
-            mappings:
-              properties:
-                '@timestamp':
-                  type: date
           data_stream:
             timestamp_field: '@timestamp'
 
@@ -153,8 +138,8 @@
 ---
 "Close write index for data stream fails":
   - skip:
-      version: " - 7.8.99"
-      reason: "data streams only supported in 7.9+"
+      version: " - 7.9.99"
+      reason: "enable in 7.9+ when backported"
       features: allowed_warnings
 
   - do:
@@ -164,11 +149,6 @@
         name: my-template1
         body:
           index_patterns: [simple-data-stream1]
-          template:
-            mappings:
-              properties:
-                '@timestamp':
-                  type: date
           data_stream:
             timestamp_field: '@timestamp'
 
@@ -190,8 +170,8 @@
 ---
 "Prohibit split on data stream's write index":
   - skip:
-      version: " - 7.8.99"
-      reason: "data streams only supported in 7.9+"
+      version: " - 7.9.99"
+      reason: "enable in 7.9+ when backported"
       features: allowed_warnings
 
   - do:
@@ -201,11 +181,6 @@
         name: my-template1
         body:
           index_patterns: [simple-data-stream1]
-          template:
-            mappings:
-              properties:
-                '@timestamp':
-                  type: date
           data_stream:
             timestamp_field: '@timestamp'
 

x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/30_auto_create_data_stream.yml

@@ -1,8 +1,8 @@
 ---
 "Put index template":
   - skip:
-      version: " - 7.8.99"
-      reason: "data streams only supported in 7.9+"
+      version: " - 7.9.99"
+      reason: "enable in 7.9+ when backported"
       features: allowed_warnings
 
   - do:
@@ -18,10 +18,6 @@
             settings:
               number_of_shards:   1
               number_of_replicas: 0
-            mappings:
-              properties:
-                '@timestamp':
-                  type: date
 
   - do:
       index:

x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/40_supported_apis.yml

@@ -2,6 +2,9 @@
 setup:
   - skip:
       features: allowed_warnings
+      version: " - 7.9.99"
+      reason: "enable in 7.9+ when backported"
+
   - do:
       allowed_warnings:
         - "index template [logs_template] has index patterns [logs-foobar] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logs_template] will take precedence during new index creation"
@@ -9,11 +12,6 @@ setup:
         name: logs_template
         body:
           index_patterns: logs-foobar
-          template:
-            mappings:
-              properties:
-                '@timestamp':
-                  type: date
           data_stream:
             timestamp_field: '@timestamp'
 
@@ -30,8 +28,8 @@ teardown:
 ---
 "Verify get index api":
   - skip:
-      version: " - 7.8.99"
-      reason: "data streams only supported in 7.9+"
+      version: " - 7.9.99"
+      reason: "enable in 7.9+ when backported"
 
   - do:
       indices.get:
@@ -67,10 +65,6 @@ teardown:
         body:
           index_patterns: [simple-data-stream1]
           template:
-            mappings:
-              properties:
-                '@timestamp':
-                  type: date
             settings:
               number_of_shards: "1"
               number_of_replicas: "0"
@@ -123,11 +117,6 @@ teardown:
         name: my-template1
         body:
           index_patterns: [simple-data-stream1]
-          template:
-            mappings:
-              properties:
-                '@timestamp':
-                  type: date
           data_stream:
             timestamp_field: '@timestamp'
 
@@ -161,11 +150,6 @@ teardown:
         name: my-template1
         body:
           index_patterns: [simple-data-stream1]
-          template:
-            mappings:
-              properties:
-                '@timestamp':
-                  type: date
           data_stream:
             timestamp_field: '@timestamp'
 
@@ -200,11 +184,6 @@ teardown:
         name: my-template1
         body:
           index_patterns: [simple-data-stream1]
-          template:
-            mappings:
-              properties:
-                '@timestamp':
-                  type: date
           data_stream:
             timestamp_field: '@timestamp'
 

x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/50_delete_backing_indices.yml

@@ -1,6 +1,8 @@
 setup:
   - skip:
       features: allowed_warnings
+      version: " - 7.9.99"
+      reason: "enable in 7.9+ when backported"
   - do:
       allowed_warnings:
         - "index template [my-template] has index patterns [simple-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation"
@@ -8,19 +10,14 @@ setup:
         name: my-template
         body:
           index_patterns: [simple-*]
-          template:
-            mappings:
-              properties:
-                '@timestamp':
-                  type: date
           data_stream:
             timestamp_field: '@timestamp'
 
 ---
 "Delete backing index on data stream":
   - skip:
-      version: " - 7.8.99"
-      reason: "data streams only supported in 7.9+"
+      version: " - 7.9.99"
+      reason: "enable in 7.9+ when backported"
 
   - do:
       indices.create_data_stream:
@@ -71,8 +68,8 @@ setup:
 ---
 "Attempt to delete write index on data stream is rejected":
   - skip:
-      version: " - 7.8.99"
-      reason:  "mute bwc until backported"
+      version: " - 7.9.99"
+      reason:  "enable in 7.9+ when backported"
 
   - do:
       indices.create_data_stream:

x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/60_get_backing_indices.yml

@@ -1,8 +1,8 @@
 ---
 "Get backing indices for data stream":
   - skip:
-      version: " - 7.8.99"
-      reason: "data streams only supported in 7.9+"
+      version: " - 7.9.99"
+      reason: "enable in 7.9+ when backported"
       features: allowed_warnings
 
   - do:
@@ -12,11 +12,6 @@
         name: my-template
         body:
           index_patterns: [data-*]
-          template:
-            mappings:
-              properties:
-                '@timestamp':
-                  type: date
           data_stream:
             timestamp_field: '@timestamp'
 

x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/70_rollover_data_streams.yml

@@ -1,8 +1,8 @@
 ---
 "Roll over a data stream":
   - skip:
-      version: " - 7.8.99"
-      reason: "data streams only supported in 7.9+"
+      version: " - 7.9.99"
+      reason: "enable in 7.9+ when backported"
       features: allowed_warnings
 
   - do:
@@ -12,11 +12,6 @@
         name: my-template
         body:
           index_patterns: [data-*]
-          template:
-            mappings:
-              properties:
-                '@timestamp':
-                  type: date
           data_stream:
             timestamp_field: '@timestamp'
 

x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/80_resolve_index_data_streams.yml

@@ -1,8 +1,8 @@
 ---
 setup:
   - skip:
-      version: "7.8.99 - "
-      reason: "resolve index api only supported in 7.9+"
+      version: " - 7.9.99"
+      reason: "enable in 7.9+ when backported"
       features: allowed_warnings
 
   - do:
@@ -12,13 +12,9 @@ setup:
         name: my-template1
         body:
           index_patterns: [simple-data-stream1]
-          template:
-            mappings:
-              properties:
-                '@timestamp':
-                  type: date
           data_stream:
             timestamp_field: '@timestamp'
+
   - do:
       allowed_warnings:
         - "index template [my-template2] has index patterns [simple-data-stream2] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template2] will take precedence during new index creation"
@@ -29,10 +25,10 @@ setup:
           template:
             mappings:
               properties:
-                '@timestamp2':
+                '@timestamp':
                   type: date
           data_stream:
-            timestamp_field: '@timestamp2'
+            timestamp_field: '@timestamp'
 
   - do:
       indices.create_data_stream:
@@ -76,8 +72,8 @@ setup:
 ---
 "Resolve index with indices, aliases, and data streams":
   - skip:
-      version: " - 7.8.99"
-      reason: "resolve index api only supported in 7.9+"
+      version: " - 7.9.99"
+      reason: "enable in 7.9+ when backported"
 
   - do:
       indices.resolve_index:
@@ -120,7 +116,7 @@ setup:
 ---
 "Resolve index with hidden and closed indices":
   - skip:
-      version: " - 7.8.99"
+      version: " - 7.9.99"
       reason: change after backporting
 
   - do:

x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/90_reindex.yml

@@ -1,8 +1,11 @@
 ---
 setup:
   - skip:
+      version: " - 7.99.99"
+      reason: "enable in 7.9+ when backported"
       features: allowed_warnings
-  - do:
+
+  - do :
       allowed_warnings:
         - "index template [generic_logs_template] has index patterns [logs-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [generic_logs_template] will take precedence during new index creation"
       indices.put_index_template:
@@ -11,11 +14,6 @@ setup:
           index_patterns: logs-*
           data_stream:
             timestamp_field: '@timestamp'
-          template:
-            mappings:
-              properties:
-                '@timestamp':
-                  type: date
 
 ---
 teardown:

x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java

@@ -67,6 +67,7 @@ import java.util.Optional;
 
 import static org.elasticsearch.action.DocWriteRequest.OpType.CREATE;
 import static org.elasticsearch.cluster.DataStreamTestHelper.generateMapping;
+import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.DEFAULT_TIMESTAMP_FIELD;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.arrayWithSize;
@@ -95,11 +96,11 @@ public class DataStreamIT extends ESIntegTestCase {
     }
 
     public void testBasicScenario() throws Exception {
-        putComposableIndexTemplate("id1", "@timestamp", List.of("metrics-foo*"));
+        putComposableIndexTemplate("id1", List.of("metrics-foo*"));
         CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request("metrics-foo");
         client().admin().indices().createDataStream(createDataStreamRequest).get();
 
-        putComposableIndexTemplate("id2", "@timestamp", List.of("metrics-bar*"));
+        putComposableIndexTemplate("id2", List.of("metrics-bar*"));
         createDataStreamRequest = new CreateDataStreamAction.Request("metrics-bar");
         client().admin().indices().createDataStream(createDataStreamRequest).get();
 
@@ -133,9 +134,9 @@ public class DataStreamIT extends ESIntegTestCase {
         assertThat(ObjectPath.eval("properties.@timestamp.type", mappings), is("date"));
 
         int numDocsBar = randomIntBetween(2, 16);
-        indexDocs("metrics-bar", "@timestamp", numDocsBar);
+        indexDocs("metrics-bar", numDocsBar);
         int numDocsFoo = randomIntBetween(2, 16);
-        indexDocs("metrics-foo", "@timestamp", numDocsFoo);
+        indexDocs("metrics-foo", numDocsFoo);
 
         verifyDocs("metrics-bar", numDocsBar, 1, 1);
         verifyDocs("metrics-foo", numDocsFoo, 1, 1);
@@ -163,9 +164,9 @@ public class DataStreamIT extends ESIntegTestCase {
         assertThat(ObjectPath.eval("properties.@timestamp.type", mappings), is("date"));
 
         int numDocsBar2 = randomIntBetween(2, 16);
-        indexDocs("metrics-bar", "@timestamp", numDocsBar2);
+        indexDocs("metrics-bar", numDocsBar2);
         int numDocsFoo2 = randomIntBetween(2, 16);
-        indexDocs("metrics-foo", "@timestamp", numDocsFoo2);
+        indexDocs("metrics-foo", numDocsFoo2);
 
         verifyDocs("metrics-bar", numDocsBar + numDocsBar2, 1, 2);
         verifyDocs("metrics-foo", numDocsFoo + numDocsFoo2, 1, 2);
@@ -206,7 +207,7 @@ public class DataStreamIT extends ESIntegTestCase {
     }
 
     public void testOtherWriteOps() throws Exception {
-        putComposableIndexTemplate("id", "@timestamp", List.of("metrics-foobar*"));
+        putComposableIndexTemplate("id", List.of("metrics-foobar*"));
         String dataStreamName = "metrics-foobar";
         CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
         client().admin().indices().createDataStream(createDataStreamRequest).get();
@@ -275,7 +276,7 @@ public class DataStreamIT extends ESIntegTestCase {
         client().execute(PutComposableIndexTemplateAction.INSTANCE, request).actionGet();
 
         int numDocs = randomIntBetween(2, 16);
-        indexDocs(dataStreamName, "@timestamp", numDocs);
+        indexDocs(dataStreamName, numDocs);
         verifyDocs(dataStreamName, numDocs, 1, 1);
 
         String backingIndex = DataStream.getDefaultBackingIndexName(dataStreamName, 1);
@@ -309,7 +310,7 @@ public class DataStreamIT extends ESIntegTestCase {
         );
 
         int numDocs2 = randomIntBetween(2, 16);
-        indexDocs(dataStreamName, "@timestamp", numDocs2);
+        indexDocs(dataStreamName, numDocs2);
         verifyDocs(dataStreamName, numDocs + numDocs2, 1, 2);
 
         DeleteDataStreamAction.Request deleteDataStreamRequest = new DeleteDataStreamAction.Request(new String[] { dataStreamName });
@@ -333,28 +334,6 @@ public class DataStreamIT extends ESIntegTestCase {
         );
     }
 
-    public void testTimeStampValidationNoFieldMapping() throws Exception {
-        // Adding a template without a mapping for timestamp field and expect template creation to fail.
-        PutComposableIndexTemplateAction.Request createTemplateRequest = new PutComposableIndexTemplateAction.Request("logs-foo");
-        createTemplateRequest.indexTemplate(
-            new ComposableIndexTemplate(
-                List.of("logs-*"),
-                new Template(null, new CompressedXContent("{}"), null),
-                null,
-                null,
-                null,
-                null,
-                new ComposableIndexTemplate.DataStreamTemplate("@timestamp")
-            )
-        );
-
-        Exception e = expectThrows(
-            IllegalArgumentException.class,
-            () -> client().execute(PutComposableIndexTemplateAction.INSTANCE, createTemplateRequest).actionGet()
-        );
-        assertThat(e.getCause().getCause().getMessage(), equalTo("the configured timestamp field [@timestamp] does not exist"));
-    }
-
     public void testTimeStampValidationInvalidFieldMapping() throws Exception {
         // Adding a template with an invalid mapping for timestamp field and expect template creation to fail.
         String mapping = "{\n"
@@ -388,7 +367,7 @@ public class DataStreamIT extends ESIntegTestCase {
     }
 
     public void testResolvabilityOfDataStreamsInAPIs() throws Exception {
-        putComposableIndexTemplate("id", "@timestamp", List.of("logs-*"));
+        putComposableIndexTemplate("id", List.of("logs-*"));
         String dataStreamName = "logs-foobar";
         CreateDataStreamAction.Request request = new CreateDataStreamAction.Request(dataStreamName);
         client().admin().indices().createDataStream(request).actionGet();
@@ -501,7 +480,7 @@ public class DataStreamIT extends ESIntegTestCase {
     }
 
     public void testCannotDeleteComposableTemplateUsedByDataStream() throws Exception {
-        putComposableIndexTemplate("id", "@timestamp", List.of("metrics-foobar*"));
+        putComposableIndexTemplate("id", List.of("metrics-foobar*"));
         String dataStreamName = "metrics-foobar-baz";
         CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
         client().admin().indices().createDataStream(createDataStreamRequest).get();
@@ -534,7 +513,7 @@ public class DataStreamIT extends ESIntegTestCase {
     }
 
     public void testAliasActionsFailOnDataStreams() throws Exception {
-        putComposableIndexTemplate("id1", "@timestamp", List.of("metrics-foo*"));
+        putComposableIndexTemplate("id1", List.of("metrics-foo*"));
         String dataStreamName = "metrics-foo";
         CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
         client().admin().indices().createDataStream(createDataStreamRequest).get();
@@ -549,7 +528,7 @@ public class DataStreamIT extends ESIntegTestCase {
     }
 
     public void testAliasActionsFailOnDataStreamBackingIndices() throws Exception {
-        putComposableIndexTemplate("id1", "@timestamp", List.of("metrics-foo*"));
+        putComposableIndexTemplate("id1", List.of("metrics-foo*"));
         String dataStreamName = "metrics-foo";
         CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
         client().admin().indices().createDataStream(createDataStreamRequest).get();
@@ -586,7 +565,7 @@ public class DataStreamIT extends ESIntegTestCase {
             + "        }\n"
             + "      }\n"
             + "    }";
-        putComposableIndexTemplate("id1", "@timestamp", mapping, List.of("logs-foo*"), null);
+        putComposableIndexTemplate("id1", mapping, List.of("logs-foo*"), null);
 
         CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request("logs-foobar");
         client().admin().indices().createDataStream(createDataStreamRequest).get();
@@ -600,7 +579,7 @@ public class DataStreamIT extends ESIntegTestCase {
     }
 
     public void testUpdateMappingViaDataStream() throws Exception {
-        putComposableIndexTemplate("id1", "@timestamp", List.of("logs-*"));
+        putComposableIndexTemplate("id1", List.of("logs-*"));
         CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request("logs-foobar");
         client().admin().indices().createDataStream(createDataStreamRequest).actionGet();
 
@@ -641,7 +620,7 @@ public class DataStreamIT extends ESIntegTestCase {
     }
 
     public void testUpdateIndexSettingsViaDataStream() throws Exception {
-        putComposableIndexTemplate("id1", "@timestamp", List.of("logs-*"));
+        putComposableIndexTemplate("id1", List.of("logs-*"));
         CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request("logs-foobar");
         client().admin().indices().createDataStream(createDataStreamRequest).actionGet();
 
@@ -670,7 +649,7 @@ public class DataStreamIT extends ESIntegTestCase {
     }
 
     public void testIndexDocsWithCustomRoutingTargetingDataStreamIsNotAllowed() throws Exception {
-        putComposableIndexTemplate("id1", "@timestamp", List.of("logs-foo*"));
+        putComposableIndexTemplate("id1", List.of("logs-foo*"));
 
         // Index doc that triggers creation of a data stream
         String dataStream = "logs-foobar";
@@ -719,7 +698,7 @@ public class DataStreamIT extends ESIntegTestCase {
     }
 
     public void testIndexDocsWithCustomRoutingTargetingBackingIndex() throws Exception {
-        putComposableIndexTemplate("id1", "@timestamp", List.of("logs-foo*"));
+        putComposableIndexTemplate("id1", List.of("logs-foo*"));
 
         // Index doc that triggers creation of a data stream
         IndexRequest indexRequest = new IndexRequest("logs-foobar").source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON)
@@ -742,25 +721,25 @@ public class DataStreamIT extends ESIntegTestCase {
     }
 
     public void testSearchAllResolvesDataStreams() throws Exception {
-        putComposableIndexTemplate("id1", "@timestamp", List.of("metrics-foo*"));
+        putComposableIndexTemplate("id1", List.of("metrics-foo*"));
         CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request("metrics-foo");
         client().admin().indices().createDataStream(createDataStreamRequest).get();
 
-        putComposableIndexTemplate("id2", "@timestamp", List.of("metrics-bar*"));
+        putComposableIndexTemplate("id2", List.of("metrics-bar*"));
         createDataStreamRequest = new CreateDataStreamAction.Request("metrics-bar");
         client().admin().indices().createDataStream(createDataStreamRequest).get();
 
         int numDocsBar = randomIntBetween(2, 16);
-        indexDocs("metrics-bar", "@timestamp", numDocsBar);
+        indexDocs("metrics-bar", numDocsBar);
         int numDocsFoo = randomIntBetween(2, 16);
-        indexDocs("metrics-foo", "@timestamp", numDocsFoo);
+        indexDocs("metrics-foo", numDocsFoo);
 
         RolloverResponse rolloverResponse = client().admin().indices().rolloverIndex(new RolloverRequest("metrics-foo", null)).get();
         assertThat(rolloverResponse.getNewIndex(), equalTo(DataStream.getDefaultBackingIndexName("metrics-foo", 2)));
 
         // ingest some more data in the rolled data stream
         int numDocsRolledFoo = randomIntBetween(2, 16);
-        indexDocs("metrics-foo", "@timestamp", numDocsRolledFoo);
+        indexDocs("metrics-foo", numDocsRolledFoo);
 
         SearchRequest searchRequest = new SearchRequest("*");
         SearchResponse searchResponse = client().search(searchRequest).actionGet();
@@ -769,10 +748,10 @@ public class DataStreamIT extends ESIntegTestCase {
 
     public void testGetDataStream() throws Exception {
         Settings settings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, maximumNumberOfReplicas() + 2).build();
-        putComposableIndexTemplate("template_for_foo", "@timestamp", List.of("metrics-foo*"), settings);
+        putComposableIndexTemplate("template_for_foo", null, List.of("metrics-foo*"), settings);
 
         int numDocsFoo = randomIntBetween(2, 16);
-        indexDocs("metrics-foo", "@timestamp", numDocsFoo);
+        indexDocs("metrics-foo", numDocsFoo);
 
         GetDataStreamAction.Response response = client().admin()
             .indices()
@@ -799,7 +778,7 @@ public class DataStreamIT extends ESIntegTestCase {
     }
 
     public void testNoTimestampInDocument() throws Exception {
-        putComposableIndexTemplate("id", "@timestamp", List.of("logs-foobar*"));
+        putComposableIndexTemplate("id", List.of("logs-foobar*"));
         String dataStreamName = "logs-foobar";
         CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
         client().admin().indices().createDataStream(createDataStreamRequest).get();
@@ -810,7 +789,7 @@ public class DataStreamIT extends ESIntegTestCase {
     }
 
     public void testMultipleTimestampValuesInDocument() throws Exception {
-        putComposableIndexTemplate("id", "@timestamp", List.of("logs-foobar*"));
+        putComposableIndexTemplate("id", List.of("logs-foobar*"));
         String dataStreamName = "logs-foobar";
         CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
         client().admin().indices().createDataStream(createDataStreamRequest).get();
@@ -946,13 +925,13 @@ public class DataStreamIT extends ESIntegTestCase {
         }
     }
 
-    private static void indexDocs(String dataStream, String timestampField, int numDocs) {
+    private static void indexDocs(String dataStream, int numDocs) {
         BulkRequest bulkRequest = new BulkRequest();
         for (int i = 0; i < numDocs; i++) {
             String value = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(System.currentTimeMillis());
             bulkRequest.add(
                 new IndexRequest(dataStream).opType(DocWriteRequest.OpType.CREATE)
-                    .source(String.format(Locale.ROOT, "{\"%s\":\"%s\"}", timestampField, value), XContentType.JSON)
+                    .source(String.format(Locale.ROOT, "{\"%s\":\"%s\"}", DEFAULT_TIMESTAMP_FIELD, value), XContentType.JSON)
             );
         }
         BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet();
@@ -980,34 +959,22 @@ public class DataStreamIT extends ESIntegTestCase {
         Arrays.stream(searchResponse.getHits().getHits()).forEach(hit -> { assertTrue(expectedIndices.contains(hit.getIndex())); });
     }
 
-    public static void putComposableIndexTemplate(String id, String timestampFieldName, List<String> patterns) throws IOException {
-        String mapping = generateMapping(timestampFieldName);
-        putComposableIndexTemplate(id, timestampFieldName, mapping, patterns, null);
+    public static void putComposableIndexTemplate(String id, List<String> patterns) throws IOException {
+        putComposableIndexTemplate(id, null, patterns, null);
     }
 
-    static void putComposableIndexTemplate(String id, String timestampFieldName, List<String> patterns, Settings settings)
+    static void putComposableIndexTemplate(String id, @Nullable String mappings, List<String> patterns, @Nullable Settings settings)
         throws IOException {
-        String mapping = generateMapping(timestampFieldName);
-        putComposableIndexTemplate(id, timestampFieldName, mapping, patterns, settings);
-    }
-
-    static void putComposableIndexTemplate(
-        String id,
-        String timestampFieldName,
-        String mapping,
-        List<String> patterns,
-        @Nullable Settings settings
-    ) throws IOException {
         PutComposableIndexTemplateAction.Request request = new PutComposableIndexTemplateAction.Request(id);
         request.indexTemplate(
             new ComposableIndexTemplate(
                 patterns,
-                new Template(settings, new CompressedXContent(mapping), null),
+                new Template(settings, mappings == null ? null : new CompressedXContent(mappings), null),
                 null,
                 null,
                 null,
                 null,
-                new ComposableIndexTemplate.DataStreamTemplate(timestampFieldName)
+                new ComposableIndexTemplate.DataStreamTemplate("@timestamp")
             )
         );
         client().execute(PutComposableIndexTemplateAction.INSTANCE, request).actionGet();

x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java

@@ -65,7 +65,7 @@ public class DataStreamsSnapshotsIT extends AbstractSnapshotIntegTestCase {
         Path location = randomRepoPath();
         createRepository(REPO, "fs", location);
 
-        DataStreamIT.putComposableIndexTemplate("t1", "@timestamp", List.of("ds", "other-ds"));
+        DataStreamIT.putComposableIndexTemplate("t1", List.of("ds", "other-ds"));
 
         CreateDataStreamAction.Request request = new CreateDataStreamAction.Request("ds");
         AcknowledgedResponse response = client.admin().indices().createDataStream(request).get();

x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/ShardClusterSnapshotRestoreIT.java

@@ -55,7 +55,7 @@ public class ShardClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCase
         );
 
         String dataStream = "datastream";
-        DataStreamIT.putComposableIndexTemplate("dst", "@timestamp", List.of(dataStream));
+        DataStreamIT.putComposableIndexTemplate("dst", List.of(dataStream));
 
         logger.info("--> indexing some data");
         for (int i = 0; i < 100; i++) {

x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java

@@ -11,7 +11,6 @@ import org.elasticsearch.client.Response;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Template;
-import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentHelper;
@@ -48,15 +47,6 @@ import static org.hamcrest.Matchers.is;
 
 public class TimeSeriesDataStreamsIT extends ESRestTestCase {
 
-    private static final String FAILED_STEP_RETRY_COUNT_FIELD = "failed_step_retry_count";
-    public static final String TIMESTAMP_MAPPING = "{\n" +
-        "      \"properties\": {\n" +
-        "        \"@timestamp\": {\n" +
-        "          \"type\": \"date\"\n" +
-        "        }\n" +
-        "      }\n" +
-        "    }";
-
     public void testRolloverAction() throws Exception {
         String policyName = "logs-policy";
         createNewSingletonPolicy(client(), policyName, "hot", new RolloverAction(null, null, 1L));
@@ -233,7 +223,7 @@ public class TimeSeriesDataStreamsIT extends ESRestTestCase {
     }
 
     private static Template getTemplate(String policyName) throws IOException {
-        return new Template(getLifcycleSettings(policyName), new CompressedXContent(TIMESTAMP_MAPPING), null);
+        return new Template(getLifcycleSettings(policyName), null, null);
     }
 
     private static Settings getLifcycleSettings(String policyName) {