Generate timestamp when path is null

Index process fails when having `_timestamp` enabled and `path` option is set.
It fails with a `TimestampParsingException[failed to parse timestamp [null]]` message.

Reproduction:

```
DELETE test
PUT  test
{
    "mappings": {
        "test": {
            "_timestamp" : {
                "enabled" : "yes",
                "path" : "post_date"
            }
        }
    }
}
PUT test/test/1
{
  "foo": "bar"
}
```

You can define a default value for when timestamp is not provided
within the index request or in the `_source` document.

By default, the default value is `now` which means the date the document was processed by the indexing chain.

You can disable that default value by setting `default` to `null`. It means that `timestamp` is mandatory:

```
{
    "tweet" : {
        "_timestamp" : {
            "enabled" : true,
            "default" : null
        }
    }
}
```

If you don't provide any timestamp value, indexation will fail.

You can also set the default value to any date respecting timestamp format:

```
{
    "tweet" : {
        "_timestamp" : {
            "enabled" : true,
            "format" : "YYYY-MM-dd",
            "default" : "1970-01-01"
        }
    }
}
```

If you don't provide any timestamp value, indexation will fail.

Closes #4718.
Closes #7036.

docs/reference/mapping/fields/timestamp-field.asciidoc

@@ -4,7 +4,7 @@
 The `_timestamp` field allows to automatically index the timestamp of a
 document. It can be provided externally via the index request or in the
 `_source`. If it is not provided externally it will be automatically set
-to the date the document was processed by the indexing chain.
+to a <<mapping-timestamp-field-default,default date>>.
 
 [float]
 ==== enabled
@@ -60,6 +60,7 @@ Note, using `path` without explicit timestamp value provided requires an
 additional (though quite fast) parsing phase.
 
 [float]
+[[mapping-timestamp-field-format]]
 ==== format
 
 You can define the <<mapping-date-format,date
@@ -80,3 +81,46 @@ format>> used to parse the provided timestamp value. For example:
 
 Note, the default format is `dateOptionalTime`. The timestamp value will
 first be parsed as a number and if it fails the format will be tried.
+
+[float]
+[[mapping-timestamp-field-default]]
+==== default
+
+You can define a default value for when timestamp is not provided
+within the index request or in the `_source` document.
+
+By default, the default value is `now` which means the date the document was processed by the indexing chain.
+
+You can disable that default value by setting `default` to `null`. It means that `timestamp` is mandatory:
+
+[source,js]
+--------------------------------------------------
+{
+    "tweet" : {
+        "_timestamp" : {
+            "enabled" : true,
+            "default" : null
+        }
+    }
+}
+--------------------------------------------------
+
+If you don't provide any timestamp value, indexation will fail.
+
+You can also set the default value to any date respecting <<mapping-timestamp-field-format,timestamp format>>:
+
+[source,js]
+--------------------------------------------------
+{
+    "tweet" : {
+        "_timestamp" : {
+            "enabled" : true,
+            "format" : "YYYY-MM-dd",
+            "default" : "1970-01-01"
+        }
+    }
+}
+--------------------------------------------------
+
+If you don't provide any timestamp value, indexation will fail.
+

src/main/java/org/elasticsearch/action/index/IndexRequest.java

@@ -23,6 +23,7 @@ import com.google.common.base.Charsets;
 import org.elasticsearch.*;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.RoutingMissingException;
+import org.elasticsearch.action.TimestampParsingException;
 import org.elasticsearch.action.support.replication.ShardReplicationOperationRequest;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
@@ -574,7 +575,9 @@ public class IndexRequest extends ShardReplicationOperationRequest<IndexRequest>
                     }
                     if (parseContext.shouldParseTimestamp()) {
                         timestamp = parseContext.timestamp();
-                        timestamp = MappingMetaData.Timestamp.parseStringTimestamp(timestamp, mappingMd.timestamp().dateTimeFormatter());
+                        if (timestamp != null) {
+                            timestamp = MappingMetaData.Timestamp.parseStringTimestamp(timestamp, mappingMd.timestamp().dateTimeFormatter());
+                        }
                     }
                 } catch (MapperParsingException e) {
                     throw e;
@@ -613,7 +616,18 @@ public class IndexRequest extends ShardReplicationOperationRequest<IndexRequest>
 
         // generate timestamp if not provided, we always have one post this stage...
         if (timestamp == null) {
-            timestamp = Long.toString(System.currentTimeMillis());
+            String defaultTimestamp = TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP;
+            if (mappingMd != null && mappingMd.timestamp() != null) {
+                defaultTimestamp = mappingMd.timestamp().defaultTimestamp();
+            }
+            if (!Strings.hasText(defaultTimestamp)) {
+                throw new TimestampParsingException("timestamp is required by mapping");
+            }
+            if (defaultTimestamp.equals(TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP)) {
+                timestamp = Long.toString(System.currentTimeMillis());
+            } else {
+                timestamp = MappingMetaData.Timestamp.parseStringTimestamp(defaultTimestamp, mappingMd.timestamp().dateTimeFormatter());
+            }
         }
     }
 

src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.cluster.metadata;
 
 import org.elasticsearch.ElasticsearchIllegalStateException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.TimestampParsingException;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
@@ -175,7 +176,8 @@ public class MappingMetaData {
         }
 
 
-        public static final Timestamp EMPTY = new Timestamp(false, null, TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT);
+        public static final Timestamp EMPTY = new Timestamp(false, null, TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT,
+                TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP);
 
         private final boolean enabled;
 
@@ -187,7 +189,9 @@ public class MappingMetaData {
 
         private final FormatDateTimeFormatter dateTimeFormatter;
 
-        public Timestamp(boolean enabled, String path, String format) {
+        private final String defaultTimestamp;
+
+        public Timestamp(boolean enabled, String path, String format, String defaultTimestamp) {
             this.enabled = enabled;
             this.path = path;
             if (path == null) {
@@ -197,6 +201,7 @@ public class MappingMetaData {
             }
             this.format = format;
             this.dateTimeFormatter = Joda.forPattern(format);
+            this.defaultTimestamp = defaultTimestamp;
         }
 
         public boolean enabled() {
@@ -219,6 +224,14 @@ public class MappingMetaData {
             return this.format;
         }
 
+        public String defaultTimestamp() {
+            return this.defaultTimestamp;
+        }
+
+        public boolean hasDefaultTimestamp() {
+            return this.defaultTimestamp != null;
+        }
+
         public FormatDateTimeFormatter dateTimeFormatter() {
             return this.dateTimeFormatter;
         }
@@ -233,6 +246,7 @@ public class MappingMetaData {
             if (enabled != timestamp.enabled) return false;
             if (format != null ? !format.equals(timestamp.format) : timestamp.format != null) return false;
             if (path != null ? !path.equals(timestamp.path) : timestamp.path != null) return false;
+            if (defaultTimestamp != null ? !defaultTimestamp.equals(timestamp.defaultTimestamp) : timestamp.defaultTimestamp != null) return false;
             if (!Arrays.equals(pathElements, timestamp.pathElements)) return false;
 
             return true;
@@ -245,6 +259,7 @@ public class MappingMetaData {
             result = 31 * result + (format != null ? format.hashCode() : 0);
             result = 31 * result + (pathElements != null ? Arrays.hashCode(pathElements) : 0);
             result = 31 * result + (dateTimeFormatter != null ? dateTimeFormatter.hashCode() : 0);
+            result = 31 * result + (defaultTimestamp != null ? defaultTimestamp.hashCode() : 0);
             return result;
         }
     }
@@ -263,7 +278,7 @@ public class MappingMetaData {
         this.source = docMapper.mappingSource();
         this.id = new Id(docMapper.idFieldMapper().path());
         this.routing = new Routing(docMapper.routingFieldMapper().required(), docMapper.routingFieldMapper().path());
-        this.timestamp = new Timestamp(docMapper.timestampFieldMapper().enabled(), docMapper.timestampFieldMapper().path(), docMapper.timestampFieldMapper().dateTimeFormatter().format());
+        this.timestamp = new Timestamp(docMapper.timestampFieldMapper().enabled(), docMapper.timestampFieldMapper().path(), docMapper.timestampFieldMapper().dateTimeFormatter().format(), docMapper.timestampFieldMapper().defaultTimestamp());
         this.hasParentField = docMapper.parentFieldMapper().active();
     }
 
@@ -328,6 +343,7 @@ public class MappingMetaData {
             boolean enabled = false;
             String path = null;
             String format = TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT;
+            String defaultTimestamp = TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP;
             Map<String, Object> timestampNode = (Map<String, Object>) withoutType.get("_timestamp");
             for (Map.Entry<String, Object> entry : timestampNode.entrySet()) {
                 String fieldName = Strings.toUnderscoreCase(entry.getKey());
@@ -338,9 +354,11 @@ public class MappingMetaData {
                     path = fieldNode.toString();
                 } else if (fieldName.equals("format")) {
                     format = fieldNode.toString();
+                } else if (fieldName.equals("default")) {
+                    defaultTimestamp = fieldNode.toString();
                 }
             }
-            this.timestamp = new Timestamp(enabled, path, format);
+            this.timestamp = new Timestamp(enabled, path, format, defaultTimestamp);
         } else {
             this.timestamp = Timestamp.EMPTY;
         }
@@ -528,6 +546,14 @@ public class MappingMetaData {
             out.writeBoolean(false);
         }
         out.writeString(mappingMd.timestamp().format());
+        if (out.getVersion().onOrAfter(Version.V_1_4_0)) {
+            if (mappingMd.timestamp().hasDefaultTimestamp()) {
+                out.writeBoolean(true);
+                out.writeString(mappingMd.timestamp().defaultTimestamp());
+            } else {
+                out.writeBoolean(false);
+            }
+        }
         out.writeBoolean(mappingMd.hasParentField());
     }
 
@@ -565,7 +591,8 @@ public class MappingMetaData {
         // routing
         Routing routing = new Routing(in.readBoolean(), in.readBoolean() ? in.readString() : null);
         // timestamp
-        Timestamp timestamp = new Timestamp(in.readBoolean(), in.readBoolean() ? in.readString() : null, in.readString());
+        Timestamp timestamp = new Timestamp(in.readBoolean(), in.readBoolean() ? in.readString() : null, in.readString(),
+                in.getVersion().onOrAfter(Version.V_1_4_0) ? (in.readBoolean() ? in.readString() : null) : TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP);
         final boolean hasParentField = in.readBoolean();
         return new MappingMetaData(type, source, id, routing, timestamp, hasParentField);
     }

src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java

@@ -70,6 +70,7 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
         public static final EnabledAttributeMapper ENABLED = EnabledAttributeMapper.UNSET_DISABLED;
         public static final String PATH = null;
         public static final FormatDateTimeFormatter DATE_TIME_FORMATTER = Joda.forPattern(DEFAULT_DATE_TIME_FORMAT);
+        public static final String DEFAULT_TIMESTAMP = "now";
     }
 
     public static class Builder extends NumberFieldMapper.Builder<Builder, TimestampFieldMapper> {
@@ -77,6 +78,7 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
         private EnabledAttributeMapper enabledState = EnabledAttributeMapper.UNSET_DISABLED;
         private String path = Defaults.PATH;
         private FormatDateTimeFormatter dateTimeFormatter = Defaults.DATE_TIME_FORMATTER;
+        private String defaultTimestamp = Defaults.DEFAULT_TIMESTAMP;
 
         public Builder() {
             super(Defaults.NAME, new FieldType(Defaults.FIELD_TYPE), Defaults.PRECISION_STEP_64_BIT);
@@ -97,6 +99,11 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
             return builder;
         }
 
+        public Builder defaultTimestamp(String defaultTimestamp) {
+            this.defaultTimestamp = defaultTimestamp;
+            return builder;
+        }
+
         @Override
         public TimestampFieldMapper build(BuilderContext context) {
             boolean roundCeil = Defaults.ROUND_CEIL;
@@ -104,7 +111,7 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
                 Settings settings = context.indexSettings();
                 roundCeil =  settings.getAsBoolean("index.mapping.date.round_ceil", settings.getAsBoolean("index.mapping.date.parse_upper_inclusive", Defaults.ROUND_CEIL));
             }
-            return new TimestampFieldMapper(fieldType, docValues, enabledState, path, dateTimeFormatter, roundCeil,
+            return new TimestampFieldMapper(fieldType, docValues, enabledState, path, dateTimeFormatter, defaultTimestamp, roundCeil,
                     ignoreMalformed(context), coerce(context), postingsProvider, docValuesProvider, normsLoading, fieldDataSettings, context.indexSettings());
         }
     }
@@ -124,6 +131,8 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
                     builder.path(fieldNode.toString());
                 } else if (fieldName.equals("format")) {
                     builder.dateTimeFormatter(parseDateTimeFormatter(builder.name(), fieldNode.toString()));
+                } else if (fieldName.equals("default")) {
+                    builder.defaultTimestamp(fieldNode == null ? null : fieldNode.toString());
                 }
             }
             return builder;
@@ -134,15 +143,16 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
     private EnabledAttributeMapper enabledState;
 
     private final String path;
+    private final String defaultTimestamp;
 
     public TimestampFieldMapper() {
-        this(new FieldType(Defaults.FIELD_TYPE), null, Defaults.ENABLED, Defaults.PATH, Defaults.DATE_TIME_FORMATTER,
+        this(new FieldType(Defaults.FIELD_TYPE), null, Defaults.ENABLED, Defaults.PATH, Defaults.DATE_TIME_FORMATTER, Defaults.DEFAULT_TIMESTAMP,
                 Defaults.ROUND_CEIL, Defaults.IGNORE_MALFORMED, Defaults.COERCE, null, null, null, null, ImmutableSettings.EMPTY);
     }
 
     protected TimestampFieldMapper(FieldType fieldType, Boolean docValues, EnabledAttributeMapper enabledState, String path,
-                                   FormatDateTimeFormatter dateTimeFormatter, boolean roundCeil,
-                                   Explicit<Boolean> ignoreMalformed,Explicit<Boolean> coerce, PostingsFormatProvider postingsProvider,
+                                   FormatDateTimeFormatter dateTimeFormatter, String defaultTimestamp, boolean roundCeil,
+                                   Explicit<Boolean> ignoreMalformed, Explicit<Boolean> coerce, PostingsFormatProvider postingsProvider,
                                    DocValuesFormatProvider docValuesProvider, Loading normsLoading,
                                    @Nullable Settings fieldDataSettings, Settings indexSettings) {
         super(new Names(Defaults.NAME, Defaults.NAME, Defaults.NAME, Defaults.NAME), dateTimeFormatter,
@@ -152,6 +162,7 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
                 indexSettings, MultiFields.empty(), null);
         this.enabledState = enabledState;
         this.path = path;
+        this.defaultTimestamp = defaultTimestamp;
     }
 
     @Override
@@ -167,6 +178,10 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
         return this.path;
     }
 
+    public String defaultTimestamp() {
+        return this.defaultTimestamp;
+    }
+
     public FormatDateTimeFormatter dateTimeFormatter() {
         return this.dateTimeFormatter;
     }
@@ -226,7 +241,8 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
         // if all are defaults, no sense to write it at all
         if (!includeDefaults && fieldType.indexed() == Defaults.FIELD_TYPE.indexed() && customFieldDataSettings == null &&
                 fieldType.stored() == Defaults.FIELD_TYPE.stored() && enabledState == Defaults.ENABLED && path == Defaults.PATH
-                && dateTimeFormatter.format().equals(Defaults.DATE_TIME_FORMATTER.format())) {
+                && dateTimeFormatter.format().equals(Defaults.DATE_TIME_FORMATTER.format())
+                && Defaults.DEFAULT_TIMESTAMP.equals(defaultTimestamp)) {
             return builder;
         }
         builder.startObject(CONTENT_TYPE);
@@ -246,6 +262,9 @@ public class TimestampFieldMapper extends DateFieldMapper implements InternalMap
             if (includeDefaults || !dateTimeFormatter.format().equals(Defaults.DATE_TIME_FORMATTER.format())) {
                 builder.field("format", dateTimeFormatter.format());
             }
+            if (includeDefaults || !Defaults.DEFAULT_TIMESTAMP.equals(defaultTimestamp)) {
+                builder.field("default", defaultTimestamp);
+            }
             if (customFieldDataSettings != null) {
                 builder.field("fielddata", (Map) customFieldDataSettings.getAsMap());
             } else if (includeDefaults) {

src/test/java/org/elasticsearch/cluster/metadata/MappingMetaDataParserTests.java

@@ -22,6 +22,7 @@ package org.elasticsearch.cluster.metadata;
 import org.elasticsearch.common.compress.CompressedString;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.mapper.MapperParsingException;
+import org.elasticsearch.index.mapper.internal.TimestampFieldMapper;
 import org.elasticsearch.test.ElasticsearchTestCase;
 import org.junit.Test;
 
@@ -36,7 +37,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
         MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
                 new MappingMetaData.Id("id"),
                 new MappingMetaData.Routing(true, "routing"),
-                new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime"), false);
+                new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
         byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
                 .field("id", "id").field("routing", "routing_value").field("timestamp", "1").endObject().bytes().toBytes();
         MappingMetaData.ParseContext parseContext = md.createParseContext(null, "routing_value", "1");
@@ -54,7 +55,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
         MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
                 new MappingMetaData.Id("id"),
                 new MappingMetaData.Routing(true, "routing"),
-                new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime"), false);
+                new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
         byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
                 .startArray("id").value("id").endArray().field("routing", "routing_value").field("timestamp", "1").endObject().bytes().toBytes();
         MappingMetaData.ParseContext parseContext = md.createParseContext(null, "routing_value", "1");
@@ -81,7 +82,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
         MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
                 new MappingMetaData.Id("id"),
                 new MappingMetaData.Routing(true, "routing"),
-                new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime"), false);
+                new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
         byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
                 .field("id", "id").field("routing", "routing_value").field("timestamp", "1").endObject().bytes().toBytes();
         MappingMetaData.ParseContext parseContext = md.createParseContext("id", null, "1");
@@ -99,7 +100,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
         MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
                 new MappingMetaData.Id("id"),
                 new MappingMetaData.Routing(true, "routing"),
-                new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime"), false);
+                new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
         byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
                 .field("id", "id").field("routing", "routing_value").field("timestamp", "1").endObject().bytes().toBytes();
         MappingMetaData.ParseContext parseContext = md.createParseContext("id", "routing_value1", null);
@@ -117,11 +118,11 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
         MappingMetaData md1 = new MappingMetaData("type1", new CompressedString(""),
                 new MappingMetaData.Id("id"),
                 new MappingMetaData.Routing(true, "routing"),
-                new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime"), false);
+                new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
         MappingMetaData md2 = new MappingMetaData("type1", new CompressedString(""),
                 new MappingMetaData.Id("id"),
                 new MappingMetaData.Routing(true, "routing"),
-                new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime"), false);
+                new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
         assertThat(md1, equalTo(md2));
     }
 
@@ -130,7 +131,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
         MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
                 new MappingMetaData.Id("id"),
                 new MappingMetaData.Routing(true, "routing"),
-                new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime"), false);
+                new MappingMetaData.Timestamp(true, "timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
         byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
                 .field("id", "id").field("routing", "routing_value").field("timestamp", "1").endObject().bytes().toBytes();
         MappingMetaData.ParseContext parseContext = md.createParseContext(null, null, null);
@@ -145,7 +146,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
         MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
                 new MappingMetaData.Id("obj1.id"),
                 new MappingMetaData.Routing(true, "obj1.routing"),
-                new MappingMetaData.Timestamp(true, "obj2.timestamp", "dateOptionalTime"), false);
+                new MappingMetaData.Timestamp(true, "obj2.timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
         byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
                 .startObject("obj0").field("field1", "value1").field("field2", "value2").endObject()
                 .startObject("obj1").field("id", "id").field("routing", "routing_value").endObject()
@@ -163,7 +164,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
         MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
                 new MappingMetaData.Id("obj1.id"),
                 new MappingMetaData.Routing(true, "obj1.routing"),
-                new MappingMetaData.Timestamp(true, "obj2.timestamp", "dateOptionalTime"), false);
+                new MappingMetaData.Timestamp(true, "obj2.timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
         byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
                 .startObject("obj0").field("field1", "value1").field("field2", "value2").endObject()
                 .startObject("obj1").field("id", "id").field("routing", "routing_value").endObject()
@@ -184,7 +185,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
         MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
                 new MappingMetaData.Id("obj1.id"),
                 new MappingMetaData.Routing(true, "obj1.routing"),
-                new MappingMetaData.Timestamp(true, "obj2.timestamp", "dateOptionalTime"), false);
+                new MappingMetaData.Timestamp(true, "obj2.timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
         byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
                 .startObject("obj0").field("field1", "value1").field("field2", "value2").endObject()
                 .startObject("obj1").field("id", "id").field("routing", "routing_value").endObject()
@@ -205,7 +206,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
         MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
                 new MappingMetaData.Id("obj1.id"),
                 new MappingMetaData.Routing(true, "obj1.routing"),
-                new MappingMetaData.Timestamp(true, "obj2.timestamp", "dateOptionalTime"), false);
+                new MappingMetaData.Timestamp(true, "obj2.timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
         byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
                 .startObject("obj0").field("field1", "value1").field("field2", "value2").endObject()
                 .startObject("obj1").field("routing", "routing_value").endObject()
@@ -226,7 +227,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
         MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
                 new MappingMetaData.Id("obj1.id"),
                 new MappingMetaData.Routing(true, "obj1.routing"),
-                new MappingMetaData.Timestamp(true, "obj1.timestamp", "dateOptionalTime"), false);
+                new MappingMetaData.Timestamp(true, "obj1.timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
         byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
                 .startObject("obj0").field("field1", "value1").field("field2", "value2").endObject()
                 .startObject("obj1").field("id", "id").field("routing", "routing_value").field("timestamp", "1").endObject()
@@ -244,7 +245,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
         MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
                 new MappingMetaData.Id("obj1.obj0.id"),
                 new MappingMetaData.Routing(true, "obj1.obj2.routing"),
-                new MappingMetaData.Timestamp(true, "obj1.obj3.timestamp", "dateOptionalTime"), false);
+                new MappingMetaData.Timestamp(true, "obj1.obj3.timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
         byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
                 .startObject("obj0").field("field1", "value1").field("field2", "value2").endObject()
                 .startObject("obj1")
@@ -273,7 +274,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
         MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
                 new MappingMetaData.Id("obj1.id"),
                 new MappingMetaData.Routing(true, "obj1.routing"),
-                new MappingMetaData.Timestamp(true, "obj1.timestamp", "dateOptionalTime"), false);
+                new MappingMetaData.Timestamp(true, "obj1.timestamp", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
         byte[] bytes = jsonBuilder().startObject().field("field1", "value1").field("field2", "value2")
                 .startObject("obj0").field("field1", "value1").field("field2", "value2").endObject()
                 .startObject("obj1").field("id", "id").endObject()
@@ -293,7 +294,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
         MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
                 new MappingMetaData.Id("field1"),
                 new MappingMetaData.Routing(true, "field1.field1"),
-                new MappingMetaData.Timestamp(true, "field1", "dateOptionalTime"), false);
+                new MappingMetaData.Timestamp(true, "field1", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
 
         byte[] bytes = jsonBuilder().startObject()
                 .field("aaa", "wr")
@@ -316,7 +317,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
         MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
                 new MappingMetaData.Id("id"),
                 new MappingMetaData.Routing(true, "field1.field1.field2"),
-                new MappingMetaData.Timestamp(true, "field1", "dateOptionalTime"), false);
+                new MappingMetaData.Timestamp(true, "field1", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
 
         byte[] bytes = jsonBuilder().startObject()
                 .field("aaa", "wr")
@@ -339,7 +340,7 @@ public class MappingMetaDataParserTests extends ElasticsearchTestCase {
         MappingMetaData md = new MappingMetaData("type1", new CompressedString(""),
                 new MappingMetaData.Id(null),
                 new MappingMetaData.Routing(true, "field1.field2"),
-                new MappingMetaData.Timestamp(true, "field1", "dateOptionalTime"), false);
+                new MappingMetaData.Timestamp(true, "field1", "dateOptionalTime", TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP), false);
 
         byte[] bytes = jsonBuilder().startObject()
                 .field("aaa", "wr")

src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java

@@ -19,7 +19,15 @@
 
 package org.elasticsearch.index.mapper.timestamp;
 
+import org.elasticsearch.action.TimestampParsingException;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.compress.CompressedString;
+import org.elasticsearch.common.io.stream.BytesStreamInput;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.joda.Joda;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -32,6 +40,7 @@ import org.elasticsearch.index.mapper.internal.TimestampFieldMapper;
 import org.elasticsearch.test.ElasticsearchSingleNodeTest;
 import org.junit.Test;
 
+import java.io.IOException;
 import java.util.Locale;
 import java.util.Map;
 
@@ -154,4 +163,246 @@ public class TimestampMappingTests extends ElasticsearchSingleNodeTest {
         assertThat(timestampConfiguration, hasKey("index"));
         assertThat(timestampConfiguration.get("index").toString(), is("no"));
     }
-}
+
+    @Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null]
+    public void testPathMissingDefaultValue() throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("_timestamp")
+                    .field("enabled", "yes")
+                    .field("path", "timestamp")
+                .endObject()
+                .endObject().endObject();
+        XContentBuilder doc = XContentFactory.jsonBuilder()
+                .startObject()
+                    .field("foo", "bar")
+                .endObject();
+
+        MetaData metaData = MetaData.builder().build();
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping.string());
+
+        MappingMetaData mappingMetaData = new MappingMetaData(docMapper);
+
+        IndexRequest request = new IndexRequest("test", "type", "1").source(doc);
+        request.process(metaData, null, mappingMetaData, true);
+        assertThat(request.timestamp(), notNullValue());
+
+        // We should have less than one minute (probably some ms)
+        long delay = System.currentTimeMillis() - Long.parseLong(request.timestamp());
+        assertThat(delay, lessThanOrEqualTo(60000L));
+    }
+
+    @Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null]
+    public void testTimestampDefaultValue() throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("_timestamp")
+                    .field("enabled", "yes")
+                .endObject()
+                .endObject().endObject();
+        XContentBuilder doc = XContentFactory.jsonBuilder()
+                .startObject()
+                    .field("foo", "bar")
+                .endObject();
+
+        MetaData metaData = MetaData.builder().build();
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping.string());
+
+        MappingMetaData mappingMetaData = new MappingMetaData(docMapper);
+
+        IndexRequest request = new IndexRequest("test", "type", "1").source(doc);
+        request.process(metaData, null, mappingMetaData, true);
+        assertThat(request.timestamp(), notNullValue());
+
+        // We should have less than one minute (probably some ms)
+        long delay = System.currentTimeMillis() - Long.parseLong(request.timestamp());
+        assertThat(delay, lessThanOrEqualTo(60000L));
+    }
+
+    @Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null]
+    public void testPathMissingDefaultToEpochValue() throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("_timestamp")
+                    .field("enabled", "yes")
+                    .field("path", "timestamp")
+                    .field("default", "1970-01-01")
+                    .field("format", "YYYY-MM-dd")
+                .endObject()
+                .endObject().endObject();
+        XContentBuilder doc = XContentFactory.jsonBuilder()
+                .startObject()
+                    .field("foo", "bar")
+                .endObject();
+
+        MetaData metaData = MetaData.builder().build();
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping.string());
+
+        MappingMetaData mappingMetaData = new MappingMetaData(docMapper);
+
+        IndexRequest request = new IndexRequest("test", "type", "1").source(doc);
+        request.process(metaData, null, mappingMetaData, true);
+        assertThat(request.timestamp(), notNullValue());
+        assertThat(request.timestamp(), is(MappingMetaData.Timestamp.parseStringTimestamp("1970-01-01", Joda.forPattern("YYYY-MM-dd"))));
+    }
+
+    @Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null]
+    public void testTimestampMissingDefaultToEpochValue() throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("_timestamp")
+                    .field("enabled", "yes")
+                    .field("default", "1970-01-01")
+                    .field("format", "YYYY-MM-dd")
+                .endObject()
+                .endObject().endObject();
+        XContentBuilder doc = XContentFactory.jsonBuilder()
+                .startObject()
+                    .field("foo", "bar")
+                .endObject();
+
+        MetaData metaData = MetaData.builder().build();
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping.string());
+
+        MappingMetaData mappingMetaData = new MappingMetaData(docMapper);
+
+        IndexRequest request = new IndexRequest("test", "type", "1").source(doc);
+        request.process(metaData, null, mappingMetaData, true);
+        assertThat(request.timestamp(), notNullValue());
+        assertThat(request.timestamp(), is(MappingMetaData.Timestamp.parseStringTimestamp("1970-01-01", Joda.forPattern("YYYY-MM-dd"))));
+    }
+
+    @Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null]
+    public void testPathMissingNowDefaultValue() throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("_timestamp")
+                    .field("enabled", "yes")
+                    .field("path", "timestamp")
+                    .field("default", "now")
+                    .field("format", "YYYY-MM-dd")
+                .endObject()
+                .endObject().endObject();
+        XContentBuilder doc = XContentFactory.jsonBuilder()
+                .startObject()
+                    .field("foo", "bar")
+                .endObject();
+
+        MetaData metaData = MetaData.builder().build();
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping.string());
+
+        MappingMetaData mappingMetaData = new MappingMetaData(docMapper);
+
+        IndexRequest request = new IndexRequest("test", "type", "1").source(doc);
+        request.process(metaData, null, mappingMetaData, true);
+        assertThat(request.timestamp(), notNullValue());
+
+        // We should have less than one minute (probably some ms)
+        long delay = System.currentTimeMillis() - Long.parseLong(request.timestamp());
+        assertThat(delay, lessThanOrEqualTo(60000L));
+    }
+
+    @Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null]
+    public void testTimestampMissingNowDefaultValue() throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("_timestamp")
+                    .field("enabled", "yes")
+                    .field("default", "now")
+                    .field("format", "YYYY-MM-dd")
+                .endObject()
+                .endObject().endObject();
+        XContentBuilder doc = XContentFactory.jsonBuilder()
+                .startObject()
+                    .field("foo", "bar")
+                .endObject();
+
+        MetaData metaData = MetaData.builder().build();
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping.string());
+
+        MappingMetaData mappingMetaData = new MappingMetaData(docMapper);
+
+        IndexRequest request = new IndexRequest("test", "type", "1").source(doc);
+        request.process(metaData, null, mappingMetaData, true);
+        assertThat(request.timestamp(), notNullValue());
+
+        // We should have less than one minute (probably some ms)
+        long delay = System.currentTimeMillis() - Long.parseLong(request.timestamp());
+        assertThat(delay, lessThanOrEqualTo(60000L));
+    }
+
+    @Test(expected = TimestampParsingException.class) // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null]
+    public void testPathMissingShouldFail() throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("_timestamp")
+                    .field("enabled", "yes")
+                    .field("path", "timestamp")
+                    .field("default", (String) null)
+                .endObject()
+                .endObject().endObject();
+        XContentBuilder doc = XContentFactory.jsonBuilder()
+                .startObject()
+                    .field("foo", "bar")
+                .endObject();
+
+        MetaData metaData = MetaData.builder().build();
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping.string());
+
+        MappingMetaData mappingMetaData = new MappingMetaData(docMapper);
+
+        IndexRequest request = new IndexRequest("test", "type", "1").source(doc);
+        request.process(metaData, null, mappingMetaData, true);
+    }
+
+    @Test(expected = TimestampParsingException.class) // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null]
+    public void testTimestampMissingShouldFail() throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("_timestamp")
+                    .field("enabled", "yes")
+                    .field("default", (String) null)
+                .endObject()
+                .endObject().endObject();
+        XContentBuilder doc = XContentFactory.jsonBuilder()
+                .startObject()
+                    .field("foo", "bar")
+                .endObject();
+
+        MetaData metaData = MetaData.builder().build();
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping.string());
+
+        MappingMetaData mappingMetaData = new MappingMetaData(docMapper);
+
+        IndexRequest request = new IndexRequest("test", "type", "1").source(doc);
+        request.process(metaData, null, mappingMetaData, true);
+    }
+
+    public void testDefaultTimestampStream() throws IOException {
+        // Testing null value for default timestamp
+        {
+            MappingMetaData.Timestamp timestamp = new MappingMetaData.Timestamp(true, null,
+                    TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT, null);
+            MappingMetaData expected = new MappingMetaData("type", new CompressedString("{}".getBytes(UTF8)),
+                    new MappingMetaData.Id(null), new MappingMetaData.Routing(false, null), timestamp, false);
+
+            BytesStreamOutput out = new BytesStreamOutput();
+            MappingMetaData.writeTo(expected, out);
+            out.close();
+            BytesReference bytes = out.bytes();
+
+            MappingMetaData metaData = MappingMetaData.readFrom(new BytesStreamInput(bytes));
+
+            assertThat(metaData, is(expected));
+        }
+
+        // Testing "now" value for default timestamp
+        {
+            MappingMetaData.Timestamp timestamp = new MappingMetaData.Timestamp(true, null,
+                    TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT, "now");
+            MappingMetaData expected = new MappingMetaData("type", new CompressedString("{}".getBytes(UTF8)),
+                    new MappingMetaData.Id(null), new MappingMetaData.Routing(false, null), timestamp, false);
+
+            BytesStreamOutput out = new BytesStreamOutput();
+            MappingMetaData.writeTo(expected, out);
+            out.close();
+            BytesReference bytes = out.bytes();
+
+            MappingMetaData metaData = MappingMetaData.readFrom(new BytesStreamInput(bytes));
+
+            assertThat(metaData, is(expected));
+        }
+    }
+}