Mappings: Store _timestamp by default.

Storing `_timestamp` by default means that under the default configuration, you
would have all the information you need in order to reindex into a different
index.

Close #8139

docs/reference/mapping/fields/timestamp-field.asciidoc

@@ -24,7 +24,7 @@ should be defined:
 [float]
 ==== store / index
 
-By default the `_timestamp` field has `store` set to `false` and `index`
+By default the `_timestamp` field has `store` set to `true` and `index`
 set to `not_analyzed`. It can be queried as a standard date field.
 
 [float]

rest-api-spec/test/create/70_timestamp.yaml

@@ -9,7 +9,6 @@
                 test:
                   _timestamp:
                     enabled:  1
-                    store:    yes
  - do:
       cluster.health:
           wait_for_status: yellow

rest-api-spec/test/create/75_ttl.yaml

@@ -12,7 +12,6 @@
                 test:
                   _ttl:
                     enabled:  1
-                    store:    yes
                     default:  10s
  - do:
       cluster.health:

rest-api-spec/test/update/70_timestamp.yaml

@@ -9,7 +9,6 @@
                 test:
                   _timestamp:
                     enabled:  1
-                    store:    yes
  - do:
       cluster.health:
           wait_for_status: yellow

rest-api-spec/test/update/85_fields_meta.yaml

@@ -14,10 +14,8 @@
                 _parent: { type: "foo" }
                 _timestamp:
                   enabled:  1
-                  store:    yes
                 _ttl:
                   enabled:  1
-                  store:    yes
                   default:  10s
 
   - do:

src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java

@@ -22,12 +22,12 @@ package org.elasticsearch.index.mapper.internal;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.NumericDocValuesField;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.joda.FormatDateTimeFormatter;
 import org.elasticsearch.common.joda.Joda;
-import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.codec.docvaluesformat.DocValuesFormatProvider;
@@ -58,13 +58,17 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
     public static class Defaults extends DateFieldMapper.Defaults {
         public static final String NAME = "_timestamp";
 
+        public static final FieldType PRE_20_FIELD_TYPE;
         public static final FieldType FIELD_TYPE = new FieldType(DateFieldMapper.Defaults.FIELD_TYPE);
 
         static {
-            FIELD_TYPE.setStored(false);
+            FIELD_TYPE.setStored(true);
             FIELD_TYPE.setIndexed(true);
             FIELD_TYPE.setTokenized(false);
             FIELD_TYPE.freeze();
+            PRE_20_FIELD_TYPE = new FieldType(FIELD_TYPE);
+            PRE_20_FIELD_TYPE.setStored(false);
+            PRE_20_FIELD_TYPE.freeze();
         }
 
         public static final EnabledAttributeMapper ENABLED = EnabledAttributeMapper.UNSET_DISABLED;
@@ -79,6 +83,7 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
         private String path = Defaults.PATH;
         private FormatDateTimeFormatter dateTimeFormatter = Defaults.DATE_TIME_FORMATTER;
         private String defaultTimestamp = Defaults.DEFAULT_TIMESTAMP;
+        private boolean explicitStore = false;
 
         public Builder() {
             super(Defaults.NAME, new FieldType(Defaults.FIELD_TYPE), Defaults.PRECISION_STEP_64_BIT);
@@ -104,8 +109,18 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
             return builder;
         }
 
+        @Override
+        public Builder store(boolean store) {
+            explicitStore = true;
+            return super.store(store);
+        }
+
         @Override
         public TimestampFieldMapper build(BuilderContext context) {
+            if (explicitStore == false && context.indexCreatedVersion().before(Version.V_2_0_0)) {
+                assert fieldType.stored();
+                fieldType.setStored(false);
+            }
             boolean roundCeil = Defaults.ROUND_CEIL;
             if (context.indexSettings() != null) {
                 Settings settings = context.indexSettings();
@@ -139,14 +154,18 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
         }
     }
 
+    private static FieldType defaultFieldType(Settings settings) {
+        return Version.indexCreated(settings).onOrAfter(Version.V_2_0_0) ? Defaults.FIELD_TYPE : Defaults.PRE_20_FIELD_TYPE;
+    }
 
     private EnabledAttributeMapper enabledState;
 
     private final String path;
     private final String defaultTimestamp;
+    private final FieldType defaultFieldType;
 
     public TimestampFieldMapper(Settings indexSettings) {
-        this(new FieldType(Defaults.FIELD_TYPE), null, Defaults.ENABLED, Defaults.PATH, Defaults.DATE_TIME_FORMATTER, Defaults.DEFAULT_TIMESTAMP,
+        this(new FieldType(defaultFieldType(indexSettings)), null, Defaults.ENABLED, Defaults.PATH, Defaults.DATE_TIME_FORMATTER, Defaults.DEFAULT_TIMESTAMP,
                 Defaults.ROUND_CEIL, Defaults.IGNORE_MALFORMED, Defaults.COERCE, null, null, null, null, indexSettings);
     }
 
@@ -163,11 +182,12 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
         this.enabledState = enabledState;
         this.path = path;
         this.defaultTimestamp = defaultTimestamp;
+        this.defaultFieldType = defaultFieldType(indexSettings);
     }
 
     @Override
     public FieldType defaultFieldType() {
-        return Defaults.FIELD_TYPE;
+        return defaultFieldType;
     }
 
     public boolean enabled() {

src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java

@@ -19,8 +19,10 @@
 
 package org.elasticsearch.index.mapper.timestamp;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.TimestampParsingException;
 import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -28,25 +30,36 @@ import org.elasticsearch.common.compress.CompressedString;
 import org.elasticsearch.common.io.stream.BytesStreamInput;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.joda.Joda;
+import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.ParsedDocument;
-import org.elasticsearch.index.mapper.SourceToParse;
 import org.elasticsearch.index.mapper.DocumentMapperParser;
 import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.index.mapper.SourceToParse;
 import org.elasticsearch.index.mapper.internal.TimestampFieldMapper;
 import org.elasticsearch.test.ElasticsearchSingleNodeTest;
+import org.elasticsearch.test.ElasticsearchTestCase;
 import org.junit.Test;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
-import static org.hamcrest.Matchers.*;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.isIn;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+import static org.hamcrest.Matchers.notNullValue;
 
 /**
  */
@@ -86,13 +99,19 @@ public class TimestampMappingTests extends ElasticsearchSingleNodeTest {
 
     @Test
     public void testDefaultValues() throws Exception {
-        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().string();
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping);
-        assertThat(docMapper.timestampFieldMapper().enabled(), equalTo(TimestampFieldMapper.Defaults.ENABLED.enabled));
-        assertThat(docMapper.timestampFieldMapper().fieldType().stored(), equalTo(TimestampFieldMapper.Defaults.FIELD_TYPE.stored()));
-        assertThat(docMapper.timestampFieldMapper().fieldType().indexed(), equalTo(TimestampFieldMapper.Defaults.FIELD_TYPE.indexed()));
-        assertThat(docMapper.timestampFieldMapper().path(), equalTo(TimestampFieldMapper.Defaults.PATH));
-        assertThat(docMapper.timestampFieldMapper().dateTimeFormatter().format(), equalTo(TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT));
+        for (Version version : Arrays.asList(Version.V_1_5_0, Version.V_2_0_0, ElasticsearchTestCase.randomVersion())) {
+            for (String mapping : Arrays.asList(
+                    XContentFactory.jsonBuilder().startObject().startObject("type").endObject().string(),
+                    XContentFactory.jsonBuilder().startObject().startObject("type").startObject("_timestamp").endObject().endObject().string())) {
+                DocumentMapper docMapper = createIndex("test", ImmutableSettings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build()).mapperService().documentMapperParser().parse(mapping);
+                assertThat(docMapper.timestampFieldMapper().enabled(), equalTo(TimestampFieldMapper.Defaults.ENABLED.enabled));
+                assertThat(docMapper.timestampFieldMapper().fieldType().stored(), equalTo(version.onOrAfter(Version.V_2_0_0) ? true : false));
+                assertThat(docMapper.timestampFieldMapper().fieldType().indexed(), equalTo(TimestampFieldMapper.Defaults.FIELD_TYPE.indexed()));
+                assertThat(docMapper.timestampFieldMapper().path(), equalTo(TimestampFieldMapper.Defaults.PATH));
+                assertThat(docMapper.timestampFieldMapper().dateTimeFormatter().format(), equalTo(TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT));
+                assertAcked(client().admin().indices().prepareDelete("test").execute().get());
+            }
+        }
     }
 
 
@@ -100,13 +119,13 @@ public class TimestampMappingTests extends ElasticsearchSingleNodeTest {
     public void testSetValues() throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("_timestamp")
-                .field("enabled", "yes").field("store", "yes").field("index", "no")
+                .field("enabled", "yes").field("store", "no").field("index", "no")
                 .field("path", "timestamp").field("format", "year")
                 .endObject()
                 .endObject().endObject().string();
         DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping);
         assertThat(docMapper.timestampFieldMapper().enabled(), equalTo(true));
-        assertThat(docMapper.timestampFieldMapper().fieldType().stored(), equalTo(true));
+        assertThat(docMapper.timestampFieldMapper().fieldType().stored(), equalTo(false));
         assertThat(docMapper.timestampFieldMapper().fieldType().indexed(), equalTo(false));
         assertThat(docMapper.timestampFieldMapper().path(), equalTo("timestamp"));
         assertThat(docMapper.timestampFieldMapper().dateTimeFormatter().format(), equalTo("year"));
@@ -144,8 +163,6 @@ public class TimestampMappingTests extends ElasticsearchSingleNodeTest {
         assertThat(serializedMap, hasKey("_timestamp"));
         assertThat(serializedMap.get("_timestamp"), instanceOf(Map.class));
         Map<String, Object> timestampConfiguration = (Map<String, Object>) serializedMap.get("_timestamp");
-        assertThat(timestampConfiguration, hasKey("store"));
-        assertThat(timestampConfiguration.get("store").toString(), is("true"));
         assertThat(timestampConfiguration, hasKey("index"));
         assertThat(timestampConfiguration.get("index").toString(), is("no"));
     }
@@ -418,7 +435,6 @@ public class TimestampMappingTests extends ElasticsearchSingleNodeTest {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("_timestamp").field("enabled", true)
                 .field("index", randomBoolean() ? "no" : "analyzed") // default is "not_analyzed" which will be omitted when building the source again
-                .field("store", true)
                 .field("path", "foo")
                 .field("default", "1970-01-01")
                 .startObject("fielddata").field("format", "doc_values").endObject()

src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterTests.java

@@ -135,21 +135,21 @@ public class UpdateMappingOnClusterTests extends ElasticsearchIntegrationTest {
     @Test
     public void testUpdateTimestamp() throws IOException {
         XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("_timestamp").field("enabled", randomBoolean()).startObject("fielddata").field("loading", "lazy").field("format", "doc_values").endObject().field("store", "yes").endObject()
+                .startObject("_timestamp").field("enabled", randomBoolean()).startObject("fielddata").field("loading", "lazy").field("format", "doc_values").endObject().field("store", "no").endObject()
                 .endObject().endObject();
         client().admin().indices().prepareCreate("test").addMapping("type", mapping).get();
         GetMappingsResponse appliedMappings = client().admin().indices().prepareGetMappings("test").get();
         LinkedHashMap timestampMapping = (LinkedHashMap) appliedMappings.getMappings().get("test").get("type").getSourceAsMap().get("_timestamp");
-        assertThat((Boolean) timestampMapping.get("store"), equalTo(true));
+        assertThat((Boolean) timestampMapping.get("store"), equalTo(false));
         assertThat((String)((LinkedHashMap) timestampMapping.get("fielddata")).get("loading"), equalTo("lazy"));
         assertThat((String)((LinkedHashMap) timestampMapping.get("fielddata")).get("format"), equalTo("doc_values"));
         mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("_timestamp").field("enabled", randomBoolean()).startObject("fielddata").field("loading", "eager").field("format", "array").endObject().field("store", "yes").endObject()
+                .startObject("_timestamp").field("enabled", randomBoolean()).startObject("fielddata").field("loading", "eager").field("format", "array").endObject().field("store", "no").endObject()
                 .endObject().endObject();
         PutMappingResponse putMappingResponse = client().admin().indices().preparePutMapping("test").setType("type").setSource(mapping).get();
         appliedMappings = client().admin().indices().prepareGetMappings("test").get();
         timestampMapping = (LinkedHashMap) appliedMappings.getMappings().get("test").get("type").getSourceAsMap().get("_timestamp");
-        assertThat((Boolean) timestampMapping.get("store"), equalTo(true));
+        assertThat((Boolean) timestampMapping.get("store"), equalTo(false));
         assertThat((String)((LinkedHashMap) timestampMapping.get("fielddata")).get("loading"), equalTo("eager"));
         assertThat((String)((LinkedHashMap) timestampMapping.get("fielddata")).get("format"), equalTo("array"));
     }