Enforce valid field mapping exists for timestamp_field in templates. (#57741)

Relates to #53100

docs/reference/indices/create-data-stream.asciidoc

@@ -14,6 +14,15 @@ template that exists with a `data_stream` definition.
 PUT _index_template/template
 {
   "index_patterns": ["my-data-stream*"],
+  "template": {
+    "mappings": {
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        }
+      }
+    }
+  },
   "data_stream": {
     "timestamp_field": "@timestamp"
   }

docs/reference/indices/delete-data-stream.asciidoc

@@ -12,6 +12,15 @@ Deletes an existing data stream along with its backing indices.
 PUT _index_template/template
 {
   "index_patterns": ["my-data-stream*"],
+  "template": {
+    "mappings": {
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        }
+      }
+    }
+  },
   "data_stream": {
     "timestamp_field": "@timestamp"
   }

docs/reference/indices/get-data-stream.asciidoc

@@ -12,6 +12,15 @@ Returns information about one or more data streams.
 PUT _index_template/template
 {
   "index_patterns": ["my-data-stream*"],
+  "template": {
+    "mappings": {
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        }
+      }
+    }
+  },
   "data_stream": {
     "timestamp_field": "@timestamp"
   }

docs/reference/indices/rollover-index.asciidoc

@@ -230,6 +230,15 @@ The API returns the following response:
 PUT _index_template/template
 {
   "index_patterns": ["my-data-stream*"],
+  "template": {
+    "mappings": {
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        }
+      }
+    }
+  },
   "data_stream": {
     "timestamp_field": "@timestamp"
   }

rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml

@@ -8,6 +8,11 @@ setup:
         name: my-template1
         body:
           index_patterns: [simple-data-stream1]
+          template:
+            mappings:
+              properties:
+                '@timestamp':
+                  type: date
           data_stream:
             timestamp_field: '@timestamp'
   - do:
@@ -17,6 +22,11 @@ setup:
         name: my-template2
         body:
           index_patterns: [simple-data-stream2]
+          template:
+            mappings:
+              properties:
+                '@timestamp2':
+                  type: date
           data_stream:
             timestamp_field: '@timestamp2'
 
@@ -215,6 +225,11 @@ setup:
         name: generic_logs_template
         body:
           index_patterns: logs-*
+          template:
+            mappings:
+              properties:
+                'timestamp':
+                  type: date
           data_stream:
             timestamp_field: timestamp
 

rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/20_unsupported_apis.yml

@@ -12,6 +12,11 @@
         name: my-template
         body:
           index_patterns: [logs-*]
+          template:
+            mappings:
+              properties:
+                '@timestamp':
+                  type: date
           data_stream:
             timestamp_field: '@timestamp'
 

rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/40_supported_apis.yml

@@ -9,6 +9,11 @@ setup:
         name: logs_template
         body:
           index_patterns: logs-foobar
+          template:
+            mappings:
+              properties:
+                '@timestamp':
+                  type: date
           data_stream:
             timestamp_field: '@timestamp'
 

rest-api-spec/src/main/resources/rest-api-spec/test/indices.delete/20_backing_indices.yml

@@ -8,6 +8,11 @@ setup:
         name: my-template
         body:
           index_patterns: [simple-*]
+          template:
+            mappings:
+              properties:
+                '@timestamp':
+                  type: date
           data_stream:
             timestamp_field: '@timestamp'
 

rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/20_backing_indices.yml

@@ -12,6 +12,11 @@
         name: my-template
         body:
           index_patterns: [data-*]
+          template:
+            mappings:
+              properties:
+                '@timestamp':
+                  type: date
           data_stream:
             timestamp_field: '@timestamp'
 

rest-api-spec/src/main/resources/rest-api-spec/test/indices.open/10_basic.yml

@@ -130,6 +130,11 @@
         name: my-template1
         body:
           index_patterns: [simple-data-stream1]
+          template:
+            mappings:
+              properties:
+                '@timestamp':
+                  type: date
           data_stream:
             timestamp_field: '@timestamp'
 

rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/50_data_streams.yml

@@ -12,6 +12,11 @@
         name: my-template
         body:
           index_patterns: [data-*]
+          template:
+            mappings:
+              properties:
+                '@timestamp':
+                  type: date
           data_stream:
             timestamp_field: '@timestamp'
 

server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java

@@ -40,8 +40,10 @@ import org.elasticsearch.action.support.replication.ReplicationRequest;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
+import org.elasticsearch.cluster.metadata.Template;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -66,6 +68,7 @@ import static org.elasticsearch.action.DocWriteRequest.OpType.CREATE;
 import static org.elasticsearch.action.DocWriteResponse.Result.CREATED;
 import static org.elasticsearch.action.DocWriteResponse.Result.UPDATED;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamServiceTests.generateMapping;
 import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.arrayWithSize;
@@ -223,7 +226,7 @@ public class BulkIntegrationIT extends ESIntegTestCase {
         createTemplateRequest.indexTemplate(
             new ComposableIndexTemplate(
                 List.of("logs-foo*"),
-                null,
+                new Template(null, new CompressedXContent(generateMapping("@timestamp")), null),
                 null, null, null, null,
                 new ComposableIndexTemplate.DataStreamTemplate("@timestamp"))
         );

server/src/internalClusterTest/java/org/elasticsearch/indices/DataStreamIT.java

@@ -47,6 +47,7 @@ import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamServiceTests;
 import org.elasticsearch.cluster.metadata.Template;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.xcontent.ObjectPath;
@@ -243,6 +244,9 @@ public class DataStreamIT extends ESIntegTestCase {
             "      \"properties\": {\n" +
             "        \"baz_field\": {\n" +
             "          \"type\": \"keyword\"\n" +
+            "        },\n" +
+            "        \"@timestamp\": {\n" +
+            "          \"type\": \"date\"\n" +
             "        }\n" +
             "      }\n" +
             "    }";
@@ -305,6 +309,46 @@ public class DataStreamIT extends ESIntegTestCase {
                 DataStream.getDefaultBackingIndexName(dataStreamName, 2))).actionGet());
     }
 
+    public void testTimeStampValidationNoFieldMapping() throws Exception {
+        // Adding a template without a mapping for timestamp field and expect template creation to fail.
+        PutComposableIndexTemplateAction.Request createTemplateRequest = new PutComposableIndexTemplateAction.Request("logs-foo");
+        createTemplateRequest.indexTemplate(
+            new ComposableIndexTemplate(
+                List.of("logs-*"),
+                new Template(null, new CompressedXContent("{}"), null),
+                null, null, null, null,
+                new ComposableIndexTemplate.DataStreamTemplate("@timestamp"))
+        );
+
+        Exception e = expectThrows(IllegalArgumentException.class,
+            () -> client().execute(PutComposableIndexTemplateAction.INSTANCE, createTemplateRequest).actionGet());
+        assertThat(e.getCause().getCause().getMessage(), equalTo("expected timestamp field [@timestamp], but found no timestamp field"));
+    }
+
+    public void testTimeStampValidationInvalidFieldMapping() throws Exception {
+        // Adding a template with an invalid mapping for timestamp field and expect template creation to fail.
+        String mapping = "{\n" +
+            "      \"properties\": {\n" +
+            "        \"@timestamp\": {\n" +
+            "          \"type\": \"keyword\"\n" +
+            "        }\n" +
+            "      }\n" +
+            "    }";
+        PutComposableIndexTemplateAction.Request createTemplateRequest = new PutComposableIndexTemplateAction.Request("logs-foo");
+        createTemplateRequest.indexTemplate(
+            new ComposableIndexTemplate(
+                List.of("logs-*"),
+                new Template(null, new CompressedXContent(mapping), null),
+                null, null, null, null,
+                new ComposableIndexTemplate.DataStreamTemplate("@timestamp"))
+        );
+
+        Exception e = expectThrows(IllegalArgumentException.class,
+            () -> client().execute(PutComposableIndexTemplateAction.INSTANCE, createTemplateRequest).actionGet());
+        assertThat(e.getCause().getCause().getMessage(), equalTo("expected timestamp field [@timestamp] to be of types " +
+            "[date, date_nanos], but instead found type [keyword]"));
+    }
+
     public void testResolvabilityOfDataStreamsInAPIs() throws Exception {
         createIndexTemplate("id", "logs-*", "ts");
         String dataStreamName = "logs-foobar";
@@ -468,7 +512,8 @@ public class DataStreamIT extends ESIntegTestCase {
         request.indexTemplate(
             new ComposableIndexTemplate(
                 List.of(pattern),
-                null,
+                new Template(null,
+                    new CompressedXContent(MetadataCreateDataStreamServiceTests.generateMapping(timestampFieldName)), null),
                 null, null, null, null,
                 new ComposableIndexTemplate.DataStreamTemplate(timestampFieldName))
         );

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java

@@ -33,15 +33,22 @@ import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.mapper.DateFieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.threadpool.ThreadPool;
 
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Locale;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicReference;
 
 public class MetadataCreateDataStreamService {
 
     private static final Logger logger = LogManager.getLogger(MetadataCreateDataStreamService.class);
+    private static final Set<String> ALLOWED_TIMESTAMPFIELD_TYPES =
+        new LinkedHashSet<>(List.of(DateFieldMapper.CONTENT_TYPE, DateFieldMapper.DATE_NANOS_CONTENT_TYPE));
 
     private final ClusterService clusterService;
     private final ActiveShardsObserver activeShardsObserver;
@@ -158,4 +165,16 @@ public class MetadataCreateDataStreamService {
         return composableIndexTemplate;
     }
 
+    public static void validateTimestampFieldMapping(String timestampFieldName, MapperService mapperService) {
+        MappedFieldType timestampFieldMapper = mapperService.fieldType(timestampFieldName);
+        if (timestampFieldMapper == null) {
+            throw new IllegalArgumentException("expected timestamp field [" + timestampFieldName + "], but found no timestamp field");
+        }
+        String type = timestampFieldMapper.typeName();
+        if (ALLOWED_TIMESTAMPFIELD_TYPES.contains(type) == false) {
+            throw new IllegalArgumentException("expected timestamp field [" + timestampFieldName + "] to be of types " +
+                ALLOWED_TIMESTAMPFIELD_TYPES + ", but instead found type [" + type  + "]");
+        }
+    }
+
 }

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java

@@ -72,6 +72,7 @@ import java.util.TreeSet;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.validateTimestampFieldMapping;
 import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.NO_LONGER_ASSIGNED;
 
 /**
@@ -1048,6 +1049,10 @@ public class MetadataIndexTemplateService {
                         // dummyMapperService.merge(MapperService.SINGLE_MAPPING_NAME, mapping, MergeReason.INDEX_TEMPLATE);
                         dummyMapperService.merge(MapperService.SINGLE_MAPPING_NAME, finalMappings, MergeReason.MAPPING_UPDATE);
                     }
+                    if (template.getDataStreamTemplate() != null) {
+                        String tsFieldName = template.getDataStreamTemplate().getTimestampField();
+                        validateTimestampFieldMapping(tsFieldName, dummyMapperService);
+                    }
                 } catch (Exception e) {
                     throw new IllegalArgumentException("invalid composite mappings for [" + templateName + "]", e);
                 }

server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java

@@ -47,13 +47,16 @@ import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.RoutingFieldMapper;
 import org.elasticsearch.index.shard.IndexEventListener;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.InvalidIndexNameException;
@@ -71,6 +74,7 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 
+import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamServiceTests.generateMapping;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
@@ -543,7 +547,14 @@ public class MetadataRolloverServiceTests extends ESTestCase {
             when(env.sharedDataFile()).thenReturn(null);
             AllocationService allocationService = mock(AllocationService.class);
             when(allocationService.reroute(any(ClusterState.class), any(String.class))).then(i -> i.getArguments()[0]);
-            IndicesService indicesService = mockIndicesServices();
+            DocumentMapper documentMapper = mock(DocumentMapper.class);
+            when(documentMapper.type()).thenReturn("_doc");
+            CompressedXContent mapping = new CompressedXContent(generateMapping(dataStream.getTimeStampField()));
+            when(documentMapper.mappingSource()).thenReturn(mapping);
+            RoutingFieldMapper routingFieldMapper = mock(RoutingFieldMapper.class);
+            when(routingFieldMapper.required()).thenReturn(false);
+            when(documentMapper.routingFieldMapper()).thenReturn(routingFieldMapper);
+            IndicesService indicesService = mockIndicesServices(documentMapper);
             IndexNameExpressionResolver mockIndexNameExpressionResolver = mock(IndexNameExpressionResolver.class);
             when(mockIndexNameExpressionResolver.resolveDateMathExpression(any())).then(returnsFirstArg());
 
@@ -688,6 +699,10 @@ public class MetadataRolloverServiceTests extends ESTestCase {
     }
 
     private IndicesService mockIndicesServices() throws Exception {
+        return mockIndicesServices(null);
+    }
+
+    private IndicesService mockIndicesServices(DocumentMapper documentMapper) throws Exception {
         /*
          * Throws Exception because Eclipse uses the lower bound for
          * CheckedFunction's exception type so it thinks the "when" call
@@ -702,7 +717,7 @@ public class MetadataRolloverServiceTests extends ESTestCase {
                 when(indexService.index()).thenReturn(indexMetadata.getIndex());
                 MapperService mapperService = mock(MapperService.class);
                 when(indexService.mapperService()).thenReturn(mapperService);
-                when(mapperService.documentMapper()).thenReturn(null);
+                when(mapperService.documentMapper()).thenReturn(documentMapper);
                 when(indexService.getIndexEventListener()).thenReturn(new IndexEventListener() {});
                 when(indexService.getIndexSortSupplier()).thenReturn(() -> null);
                 //noinspection unchecked

server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java

@@ -69,12 +69,14 @@ public class ComposableIndexTemplateTests extends AbstractDiffableSerializationT
         CompressedXContent mappings = null;
         Map<String, AliasMetadata> aliases = null;
         Template template = null;
-        if (randomBoolean()) {
+        ComposableIndexTemplate.DataStreamTemplate dataStreamTemplate = randomDataStreamTemplate();
+
+        if (dataStreamTemplate != null || randomBoolean()) {
             if (randomBoolean()) {
                 settings = randomSettings();
             }
-            if (randomBoolean()) {
-                mappings = randomMappings();
+            if (dataStreamTemplate != null || randomBoolean()) {
+                mappings = randomMappings(dataStreamTemplate);
             }
             if (randomBoolean()) {
                 aliases = randomAliases();
@@ -87,8 +89,6 @@ public class ComposableIndexTemplateTests extends AbstractDiffableSerializationT
             meta = randomMeta();
         }
 
-        ComposableIndexTemplate.DataStreamTemplate dataStreamTemplate = randomDataStreamTemplate();
-
         List<String> indexPatterns = randomList(1, 4, () -> randomAlphaOfLength(4));
         List<String> componentTemplates = randomList(0, 10, () -> randomAlphaOfLength(5));
         return new ComposableIndexTemplate(indexPatterns,
@@ -111,9 +111,13 @@ public class ComposableIndexTemplateTests extends AbstractDiffableSerializationT
         return Collections.singletonMap(aliasName, aliasMeta);
     }
 
-    private static CompressedXContent randomMappings() {
+    private static CompressedXContent randomMappings(ComposableIndexTemplate.DataStreamTemplate dataStreamTemplate) {
         try {
-            return new CompressedXContent("{\"properties\":{\"" + randomAlphaOfLength(5) + "\":{\"type\":\"keyword\"}}}");
+            if (dataStreamTemplate != null) {
+                return new CompressedXContent("{\"properties\":{\"" + dataStreamTemplate.getTimestampField() + "\":{\"type\":\"date\"}}}");
+            } else {
+                return new CompressedXContent("{\"properties\":{\"" + randomAlphaOfLength(5) + "\":{\"type\":\"keyword\"}}}");
+            }
         } catch (IOException e) {
             fail("got an IO exception creating fake mappings: " + e);
             return null;
@@ -162,7 +166,8 @@ public class ComposableIndexTemplateTests extends AbstractDiffableSerializationT
                     orig.priority(), orig.version(), orig.metadata(), orig.getDataStreamTemplate());
             case 1:
                 return new ComposableIndexTemplate(orig.indexPatterns(),
-                    randomValueOtherThan(orig.template(), () -> new Template(randomSettings(), randomMappings(), randomAliases())),
+                    randomValueOtherThan(orig.template(), () -> new Template(randomSettings(),
+                        randomMappings(orig.getDataStreamTemplate()), randomAliases())),
                     orig.composedOf(),
                     orig.priority(),
                     orig.version(),

server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java

@@ -25,12 +25,16 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.CreateDataStreamClusterStateUpdateRequest;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.MapperTestUtils;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.test.ESTestCase;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
 import static org.elasticsearch.cluster.DataStreamTestHelper.createFirstBackingIndex;
+import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.validateTimestampFieldMapping;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
@@ -146,6 +150,59 @@ public class MetadataCreateDataStreamServiceTests extends ESTestCase {
         return MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req);
     }
 
+    public void testValidateTimestampFieldMapping() throws Exception {
+        String mapping = generateMapping("@timestamp", "date");
+        validateTimestampFieldMapping("@timestamp", createMapperService(mapping));
+        mapping = generateMapping("@timestamp", "date_nanos");
+        validateTimestampFieldMapping("@timestamp", createMapperService(mapping));
+    }
+
+    public void testValidateTimestampFieldMappingNoFieldMapping() {
+        Exception e = expectThrows(IllegalArgumentException.class,
+            () -> validateTimestampFieldMapping("@timestamp", createMapperService("{}")));
+        assertThat(e.getMessage(),
+            equalTo("expected timestamp field [@timestamp], but found no timestamp field"));
+
+        String mapping = generateMapping("@timestamp2", "date");
+        e = expectThrows(IllegalArgumentException.class,
+            () -> validateTimestampFieldMapping("@timestamp", createMapperService(mapping)));
+        assertThat(e.getMessage(),
+            equalTo("expected timestamp field [@timestamp], but found no timestamp field"));
+    }
+
+    public void testValidateTimestampFieldMappingInvalidFieldType() {
+        String mapping = generateMapping("@timestamp", "keyword");
+        Exception e = expectThrows(IllegalArgumentException.class,
+            () -> validateTimestampFieldMapping("@timestamp", createMapperService(mapping)));
+        assertThat(e.getMessage(), equalTo("expected timestamp field [@timestamp] to be of types [date, date_nanos], " +
+            "but instead found type [keyword]"));
+    }
+
+    public void testValidateNestedTimestampFieldMapping() throws Exception {
+        String fieldType = randomBoolean() ? "date" : "date_nanos";
+        String mapping = "{\n" +
+            "      \"properties\": {\n" +
+            "        \"event\": {\n" +
+            "          \"properties\": {\n" +
+            "             \"@timestamp\": {\n" +
+            "               \"type\": \"" + fieldType + "\"\n" +
+            "              },\n" +
+            "             \"another_field\": {\n" +
+            "               \"type\": \"keyword\"\n" +
+            "              }\n" +
+            "            }\n" +
+            "        }\n" +
+            "      }\n" +
+            "    }";
+        MapperService mapperService = createMapperService(mapping);
+
+        validateTimestampFieldMapping("event.@timestamp", mapperService);
+        Exception e = expectThrows(IllegalArgumentException.class,
+            () -> validateTimestampFieldMapping("event.another_field", mapperService));
+        assertThat(e.getMessage(), equalTo("expected timestamp field [event.another_field] to be of types [date, date_nanos], " +
+            "but instead found type [keyword]"));
+    }
+
     private static MetadataCreateIndexService getMetadataCreateIndexService() throws Exception {
         MetadataCreateIndexService s = mock(MetadataCreateIndexService.class);
         when(s.applyCreateIndexRequest(any(ClusterState.class), any(CreateIndexClusterStateUpdateRequest.class), anyBoolean()))
@@ -159,6 +216,7 @@ public class MetadataCreateDataStreamServiceTests extends ESTestCase {
                             .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
                             .put(request.settings())
                             .build())
+                        .putMapping(generateMapping("@timestamp"))
                         .numberOfShards(1)
                         .numberOfReplicas(1)
                         .build(), false);
@@ -168,4 +226,32 @@ public class MetadataCreateDataStreamServiceTests extends ESTestCase {
         return s;
     }
 
+    public static String generateMapping(String timestampFieldName) {
+        return generateMapping(timestampFieldName, "date");
+    }
+
+    static String generateMapping(String timestampFieldName, String type) {
+        return "{\n" +
+            "      \"properties\": {\n" +
+            "        \"" + timestampFieldName + "\": {\n" +
+            "          \"type\": \"" + type + "\"\n" +
+            "        }\n" +
+            "      }\n" +
+            "    }";
+    }
+
+    MapperService createMapperService(String mapping) throws IOException {
+        String indexName = "test";
+        IndexMetadata indexMetadata = IndexMetadata.builder(indexName)
+            .settings(Settings.builder()
+                .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
+                .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1))
+            .putMapping(mapping)
+            .build();
+        MapperService mapperService =
+            MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), Settings.EMPTY, indexName);
+        mapperService.merge(indexMetadata, MapperService.MergeReason.MAPPING_UPDATE);
+        return mapperService;
+    }
 }

x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java

@@ -9,6 +9,7 @@ package org.elasticsearch.xpack.ilm;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Template;
+import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.test.rest.ESRestTestCase;
@@ -42,8 +43,15 @@ public class TimeSeriesDataStreamsIT extends ESRestTestCase {
         String policyName = "logs-policy";
         createNewSingletonPolicy(client(), policyName, "hot", new RolloverAction(null, null, 1L));
 
+        String mapping = "{\n" +
+            "      \"properties\": {\n" +
+            "        \"@timestamp\": {\n" +
+            "          \"type\": \"date\"\n" +
+            "        }\n" +
+            "      }\n" +
+            "    }";
         Settings lifecycleNameSetting = Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policyName).build();
-        Template template = new Template(lifecycleNameSetting, null, null);
+        Template template = new Template(lifecycleNameSetting, new CompressedXContent(mapping), null);
         createComposableTemplate(client(), "logs-template", "logs-foo*", template);
 
         String dataStream = "logs-foo";
@@ -60,11 +68,18 @@ public class TimeSeriesDataStreamsIT extends ESRestTestCase {
         String policyName = "logs-policy";
         createNewSingletonPolicy(client(), policyName, "warm", new ShrinkAction(1));
 
+        String mapping = "{\n" +
+            "      \"properties\": {\n" +
+            "        \"@timestamp\": {\n" +
+            "          \"type\": \"date\"\n" +
+            "        }\n" +
+            "      }\n" +
+            "    }";
         Settings settings = Settings.builder()
             .put(LifecycleSettings.LIFECYCLE_NAME, policyName)
             .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3)
             .build();
-        Template template = new Template(settings, null, null);
+        Template template = new Template(settings, new CompressedXContent(mapping), null);
         createComposableTemplate(client(), "logs-template", "logs-foo*", template);
 
         String dataStream = "logs-foo";
@@ -89,11 +104,18 @@ public class TimeSeriesDataStreamsIT extends ESRestTestCase {
         String policyName = "logs-policy";
         createFullPolicy(client(), policyName, TimeValue.ZERO);
 
+        String mapping = "{\n" +
+            "      \"properties\": {\n" +
+            "        \"@timestamp\": {\n" +
+            "          \"type\": \"date\"\n" +
+            "        }\n" +
+            "      }\n" +
+            "    }";
         Settings settings = Settings.builder()
             .put(LifecycleSettings.LIFECYCLE_NAME, policyName)
             .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3)
             .build();
-        Template template = new Template(settings, null, null);
+        Template template = new Template(settings, new CompressedXContent(mapping), null);
         createComposableTemplate(client(), "logs-template", "logs-foo*", template);
 
         String dataStream = "logs-foo";
@@ -115,11 +137,18 @@ public class TimeSeriesDataStreamsIT extends ESRestTestCase {
         String policyName = "logs-policy";
         createNewSingletonPolicy(client(), policyName, "cold", new SearchableSnapshotAction(snapshotRepo));
 
+        String mapping = "{\n" +
+            "      \"properties\": {\n" +
+            "        \"@timestamp\": {\n" +
+            "          \"type\": \"date\"\n" +
+            "        }\n" +
+            "      }\n" +
+            "    }";
         Settings settings = Settings.builder()
             .put(LifecycleSettings.LIFECYCLE_NAME, policyName)
             .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3)
             .build();
-        Template template = new Template(settings, null, null);
+        Template template = new Template(settings, new CompressedXContent(mapping), null);
         createComposableTemplate(client(), "logs-template", "logs-foo*", template);
         String dataStream = "logs-foo";
         indexDocument(client(), dataStream, true);

x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml

@@ -12,6 +12,11 @@
         name: my-template
         body:
           index_patterns: [logs-*]
+          template:
+            mappings:
+              properties:
+                '@timestamp':
+                  type: date
           data_stream:
             timestamp_field: '@timestamp'