Add index-time scripts to date field mapper (#71633)

This commit allows you to set 'script' and 'on_script_error' parameters
on date field mappers, meaning that runtime date fields can be made indexed
simply by moving their definitions from the runtime section of the mappings
to the properties section.

docs/reference/mapping/types/date.asciidoc

@@ -132,7 +132,8 @@ The following parameters are accepted by `date` fields:
 <<ignore-malformed,`ignore_malformed`>>::
 
     If `true`, malformed numbers are ignored. If `false` (default), malformed
-    numbers throw an exception and reject the whole document.
+    numbers throw an exception and reject the whole document.  Note that this
+    cannot be set if the `script` parameter is used.
 
 <<mapping-index,`index`>>::
 
@@ -142,7 +143,29 @@ The following parameters are accepted by `date` fields:
 
     Accepts a date value in one of the configured +format+'s as the field
     which is substituted for any explicit `null` values. Defaults to `null`,
-    which means the field is treated as missing.
+    which means the field is treated as missing.  Note that this cannot be
+    set of the `script` parameter is used.
+
+
+`on_script_error`::
+
+    Defines what to do if the script defined by the `script` parameter
+    throws an error at indexing time. Accepts `reject` (default), which
+    will cause the entire document to be rejected, and `ignore`, which
+    will register the field in the document's
+    <<mapping-ignored-field,`_ignored`>> metadata field and continue
+    indexing. This parameter can only be set if the `script` field is
+    also set.
+
+`script`::
+
+    If this parameter is set, then the field will index values generated
+    by this script, rather than reading the values directly from the
+    source. If a value is set for this field on the input document, then
+    the document will be rejected with an error.
+    Scripts are in the same format as their
+    <<runtime-mapping-fields,runtime equivalent>>, and should emit
+    long-valued timestamps.
 
 <<mapping-store,`store`>>::
 

modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/43_date_calculated_at_index.yml

@@ -0,0 +1,178 @@
+---
+setup:
+  - do:
+      indices.create:
+        index: sensor
+        body:
+          settings:
+            number_of_shards: 1
+            number_of_replicas: 0
+          mappings:
+            properties:
+              tomorrow:
+                type: date
+                script:
+                  source: |
+                    for (def dt : doc['timestamp']) {
+                      emit(dt.plus(params.days, ChronoUnit.DAYS).toEpochMilli());
+                    }
+                  params:
+                    days: 1
+              # Test fetching from _source and parsing
+              tomorrow_from_source:
+                type: date
+                script:
+                  source: |
+                    Instant instant = Instant.ofEpochMilli(parse(params._source.timestamp));
+                    ZonedDateTime dt = ZonedDateTime.ofInstant(instant, ZoneId.of("UTC"));
+                    emit(dt.plus(1, ChronoUnit.DAYS).toEpochMilli());
+              # Test returning millis
+              the_past:
+                type: date
+                script:
+                  source: |
+                    for (def dt : doc['timestamp']) {
+                      emit(dt.toInstant().toEpochMilli() - 1000);
+                    }
+              # Test fetching many values
+              all_week:
+                type: date
+                script:
+                  source: |
+                    for (def dt : doc['timestamp']) {
+                      for (int i = 0; i < 7; i++) {
+                        emit(dt.plus(i, ChronoUnit.DAYS).toEpochMilli());
+                      }
+                    }
+              # Test format parameter
+              formatted_tomorrow:
+                type: date
+                format: yyyy-MM-dd
+                script: |
+                  for (def dt : doc['timestamp']) {
+                    emit(dt.plus(1, ChronoUnit.DAYS).toEpochMilli());
+                  }
+              timestamp:
+                type: date
+              temperature:
+                type: long
+              voltage:
+                type: double
+              node:
+                type: keyword
+
+  - do:
+      bulk:
+        index: sensor
+        refresh: true
+        body: |
+          {"index":{}}
+          {"timestamp": 1516729294000, "temperature": 200, "voltage": 5.2, "node": "a"}
+          {"index":{}}
+          {"timestamp": 1516642894000, "temperature": 201, "voltage": 5.8, "node": "b"}
+          {"index":{}}
+          {"timestamp": 1516556494000, "temperature": 202, "voltage": 5.1, "node": "a"}
+          {"index":{}}
+          {"timestamp": 1516470094000, "temperature": 198, "voltage": 5.6, "node": "b"}
+          {"index":{}}
+          {"timestamp": 1516383694000, "temperature": 200, "voltage": 4.2, "node": "c"}
+          {"index":{}}
+          {"timestamp": "2018-01-18T17:41:34.000Z", "temperature": 202, "voltage": 4.0, "node": "c"}
+
+---
+"get mapping":
+  - do:
+      indices.get_mapping:
+        index: sensor
+  - match: {sensor.mappings.properties.tomorrow.type: date }
+  - match:
+      sensor.mappings.properties.tomorrow.script.source: |
+        for (def dt : doc['timestamp']) {
+          emit(dt.plus(params.days, ChronoUnit.DAYS).toEpochMilli());
+        }
+  - match: {sensor.mappings.properties.tomorrow.script.params: {days: 1} }
+  - match: {sensor.mappings.properties.tomorrow.script.lang: painless }
+
+  - match: {sensor.mappings.properties.formatted_tomorrow.type: date }
+  - match:
+      sensor.mappings.properties.formatted_tomorrow.script.source: |
+        for (def dt : doc['timestamp']) {
+          emit(dt.plus(1, ChronoUnit.DAYS).toEpochMilli());
+        }
+  - match: {sensor.mappings.properties.formatted_tomorrow.script.lang: painless }
+  - match: {sensor.mappings.properties.formatted_tomorrow.format: yyyy-MM-dd }
+
+---
+"fetch fields":
+  - do:
+      search:
+        index: sensor
+        body:
+          sort: timestamp
+          fields: [tomorrow, tomorrow_from_source, the_past, all_week, formatted_tomorrow]
+  - match: {hits.total.value: 6}
+  - match: {hits.hits.0.fields.tomorrow: ["2018-01-19T17:41:34.000Z"] }
+  - match: {hits.hits.0.fields.tomorrow_from_source: ["2018-01-19T17:41:34.000Z"] }
+  - match: {hits.hits.0.fields.the_past: ["2018-01-18T17:41:33.000Z"] }
+  - match:
+      hits.hits.0.fields.all_week:
+        - 2018-01-18T17:41:34.000Z
+        - 2018-01-19T17:41:34.000Z
+        - 2018-01-20T17:41:34.000Z
+        - 2018-01-21T17:41:34.000Z
+        - 2018-01-22T17:41:34.000Z
+        - 2018-01-23T17:41:34.000Z
+        - 2018-01-24T17:41:34.000Z
+  - match: {hits.hits.0.fields.formatted_tomorrow: [2018-01-19] }
+
+---
+"docvalue_fields":
+  - do:
+      search:
+        index: sensor
+        body:
+          sort: timestamp
+          docvalue_fields: [tomorrow, tomorrow_from_source, the_past, all_week, formatted_tomorrow]
+  - match: {hits.total.value: 6}
+  - match: {hits.hits.0.fields.tomorrow: ["2018-01-19T17:41:34.000Z"] }
+  - match: {hits.hits.0.fields.tomorrow_from_source: ["2018-01-19T17:41:34.000Z"] }
+  - match: {hits.hits.0.fields.the_past: ["2018-01-18T17:41:33.000Z"] }
+  - match:
+      hits.hits.0.fields.all_week:
+        - 2018-01-18T17:41:34.000Z
+        - 2018-01-19T17:41:34.000Z
+        - 2018-01-20T17:41:34.000Z
+        - 2018-01-21T17:41:34.000Z
+        - 2018-01-22T17:41:34.000Z
+        - 2018-01-23T17:41:34.000Z
+        - 2018-01-24T17:41:34.000Z
+  - match: {hits.hits.0.fields.formatted_tomorrow: [2018-01-19] }
+
+---
+"terms agg":
+  - do:
+      search:
+        index: sensor
+        body:
+          aggs:
+            v10:
+              terms:
+                field: tomorrow
+                format: strict_date_optional_time
+  - match: {hits.total.value: 6}
+  - match: {aggregations.v10.buckets.0.key_as_string: "2018-01-19T17:41:34.000Z"}
+  - match: {aggregations.v10.buckets.0.doc_count: 1}
+  - match: {aggregations.v10.buckets.1.key_as_string: "2018-01-20T17:41:34.000Z"}
+  - match: {aggregations.v10.buckets.1.doc_count: 1}
+
+---
+"term query":
+  - do:
+      search:
+        index: sensor
+        body:
+          query:
+            term:
+              tomorrow: 2018-01-19T17:41:34Z
+  - match: {hits.total.value: 1}
+  - match: {hits.hits.0._source.voltage: 4.0}

server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java

@@ -12,6 +12,7 @@ import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.PointValues;
 import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery;
@@ -35,7 +36,11 @@ import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData;
 import org.elasticsearch.index.query.DateRangeIncludingNowQuery;
 import org.elasticsearch.index.query.QueryRewriteContext;
 import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.script.DateFieldScript;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptCompiler;
 import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.lookup.FieldValues;
 import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.io.IOException;
@@ -49,6 +54,7 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Objects;
 import java.util.function.BiFunction;
 import java.util.function.Function;
 import java.util.function.LongSupplier;
@@ -226,17 +232,26 @@ public final class DateFieldMapper extends FieldMapper {
             = Parameter.stringParam("null_value", false, m -> toType(m).nullValueAsString, null).acceptsNull();
         private final Parameter<Boolean> ignoreMalformed;
 
+        private final Parameter<Script> script = Parameter.scriptParam(m -> toType(m).script);
+        private final Parameter<String> onScriptError = Parameter.onScriptErrorParam(m -> toType(m).onScriptError, script);
+
         private final Resolution resolution;
         private final Version indexCreatedVersion;
+        private final ScriptCompiler scriptCompiler;
 
         public Builder(String name, Resolution resolution, DateFormatter dateFormatter,
+                       ScriptCompiler scriptCompiler,
                        boolean ignoreMalformedByDefault, Version indexCreatedVersion) {
             super(name);
             this.resolution = resolution;
             this.indexCreatedVersion = indexCreatedVersion;
+            this.scriptCompiler = Objects.requireNonNull(scriptCompiler);
             this.ignoreMalformed
                 = Parameter.boolParam("ignore_malformed", true, m -> toType(m).ignoreMalformed, ignoreMalformedByDefault);
 
+            this.script.precludesParameters(nullValue, ignoreMalformed);
+            addScriptValidation(script, index, docValues);
+
             DateFormatter defaultFormat = resolution == Resolution.MILLISECONDS ?
                 DEFAULT_DATE_TIME_FORMATTER : DEFAULT_DATE_TIME_NANOS_FORMATTER;
             this.format = Parameter.stringParam("format", false, m -> toType(m).format, defaultFormat.pattern());
@@ -254,9 +269,20 @@ public final class DateFieldMapper extends FieldMapper {
             }
         }
 
+        private FieldValues<Long> scriptValues() {
+            if (script.get() == null) {
+                return null;
+            }
+            DateFieldScript.Factory factory = scriptCompiler.compile(script.get(), DateFieldScript.CONTEXT);
+            return factory == null ? null : (lookup, ctx, doc, consumer) -> factory
+                .newFactory(name, script.get().getParams(), lookup, buildFormatter())
+                .newInstance(ctx)
+                .runForDoc(doc, consumer::accept);
+        }
+
         @Override
         protected List<Parameter<?>> getParameters() {
-            return List.of(index, docValues, store, format, locale, nullValue, ignoreMalformed, meta);
+            return List.of(index, docValues, store, format, locale, nullValue, ignoreMalformed, script, onScriptError, meta);
         }
 
         private Long parseNullValue(DateFieldType fieldType) {
@@ -280,7 +306,7 @@ public final class DateFieldMapper extends FieldMapper {
         @Override
         public DateFieldMapper build(ContentPath contentPath) {
             DateFieldType ft = new DateFieldType(buildFullName(contentPath), index.getValue(), store.getValue(), docValues.getValue(),
-                buildFormatter(), resolution, nullValue.getValue(), meta.getValue());
+                buildFormatter(), resolution, nullValue.getValue(), scriptValues(), meta.getValue());
             Long nullTimestamp = parseNullValue(ft);
             return new DateFieldMapper(name, ft, multiFieldsBuilder.build(this, contentPath),
                 copyTo.build(), nullTimestamp, resolution, this);
@@ -289,12 +315,14 @@ public final class DateFieldMapper extends FieldMapper {
 
     public static final TypeParser MILLIS_PARSER = new TypeParser((n, c) -> {
         boolean ignoreMalformedByDefault = IGNORE_MALFORMED_SETTING.get(c.getSettings());
-        return new Builder(n, Resolution.MILLISECONDS, c.getDateFormatter(), ignoreMalformedByDefault, c.indexVersionCreated());
+        return new Builder(n, Resolution.MILLISECONDS, c.getDateFormatter(), c.scriptCompiler(),
+            ignoreMalformedByDefault, c.indexVersionCreated());
     });
 
     public static final TypeParser NANOS_PARSER = new TypeParser((n, c) -> {
         boolean ignoreMalformedByDefault = IGNORE_MALFORMED_SETTING.get(c.getSettings());
-        return new Builder(n, Resolution.NANOSECONDS, c.getDateFormatter(), ignoreMalformedByDefault, c.indexVersionCreated());
+        return new Builder(n, Resolution.NANOSECONDS, c.getDateFormatter(), c.scriptCompiler(),
+            ignoreMalformedByDefault, c.indexVersionCreated());
     });
 
     public static final class DateFieldType extends MappedFieldType {
@@ -302,31 +330,33 @@ public final class DateFieldMapper extends FieldMapper {
         protected final DateMathParser dateMathParser;
         protected final Resolution resolution;
         protected final String nullValue;
+        protected final FieldValues<Long> scriptValues;
 
         public DateFieldType(String name, boolean isSearchable, boolean isStored, boolean hasDocValues,
                              DateFormatter dateTimeFormatter, Resolution resolution, String nullValue,
-                             Map<String, String> meta) {
+                             FieldValues<Long> scriptValues, Map<String, String> meta) {
             super(name, isSearchable, isStored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_WITHOUT_TERMS, meta);
             this.dateTimeFormatter = dateTimeFormatter;
             this.dateMathParser = dateTimeFormatter.toDateMathParser();
             this.resolution = resolution;
             this.nullValue = nullValue;
+            this.scriptValues = scriptValues;
         }
 
         public DateFieldType(String name) {
-            this(name, true, false, true, DEFAULT_DATE_TIME_FORMATTER, Resolution.MILLISECONDS, null, Collections.emptyMap());
+            this(name, true, false, true, DEFAULT_DATE_TIME_FORMATTER, Resolution.MILLISECONDS, null, null, Collections.emptyMap());
         }
 
         public DateFieldType(String name, DateFormatter dateFormatter) {
-            this(name, true, false, true, dateFormatter, Resolution.MILLISECONDS, null, Collections.emptyMap());
+            this(name, true, false, true, dateFormatter, Resolution.MILLISECONDS, null, null, Collections.emptyMap());
         }
 
         public DateFieldType(String name, Resolution resolution) {
-            this(name, true, false, true, DEFAULT_DATE_TIME_FORMATTER, resolution, null, Collections.emptyMap());
+            this(name, true, false, true, DEFAULT_DATE_TIME_FORMATTER, resolution, null, null, Collections.emptyMap());
         }
 
         public DateFieldType(String name, Resolution resolution, DateFormatter dateFormatter) {
-            this(name, true, false, true, dateFormatter, resolution, null, Collections.emptyMap());
+            this(name, true, false, true, dateFormatter, resolution, null, null, Collections.emptyMap());
         }
 
         @Override
@@ -368,25 +398,31 @@ public final class DateFieldMapper extends FieldMapper {
             NUMBER_FORMAT.setGroupingUsed(false);
             NUMBER_FORMAT.setMaximumFractionDigits(6);
         }
+
         @Override
         public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
             DateFormatter defaultFormatter = dateTimeFormatter();
             DateFormatter formatter = format != null
                 ? DateFormatter.forPattern(format).withLocale(defaultFormatter.locale())
                 : defaultFormatter;
-
+            if (scriptValues != null) {
+                return FieldValues.valueFetcher(scriptValues, v -> format((long)v, formatter), context);
+            }
             return new SourceValueFetcher(name(), context, nullValue) {
                 @Override
                 public String parseSourceValue(Object value) {
                     String date = value instanceof Number ? NUMBER_FORMAT.format(value) : value.toString();
                     // TODO can we emit a warning if we're losing precision here? I'm not sure we can.
-                    long timestamp = parse(date);
-                    ZonedDateTime dateTime = resolution().toInstant(timestamp).atZone(ZoneOffset.UTC);
-                    return formatter.format(dateTime);
+                    return format(parse(date), formatter);
                 }
             };
         }
 
+        private String format(long timestamp, DateFormatter formatter) {
+            ZonedDateTime dateTime = resolution().toInstant(timestamp).atZone(ZoneOffset.UTC);
+            return formatter.format(dateTime);
+        }
+
         @Override
         public Query termQuery(Object value, @Nullable SearchExecutionContext context) {
             return rangeQuery(value, value, true, true, ShapeRelation.INTERSECTS, null, null, context);
@@ -608,6 +644,10 @@ public final class DateFieldMapper extends FieldMapper {
     private final boolean ignoreMalformedByDefault;
     private final Version indexCreatedVersion;
 
+    private final Script script;
+    private final ScriptCompiler scriptCompiler;
+    private final FieldValues<Long> scriptValues;
+
     private DateFieldMapper(
             String simpleName,
             MappedFieldType mappedFieldType,
@@ -616,7 +656,7 @@ public final class DateFieldMapper extends FieldMapper {
             Long nullValue,
             Resolution resolution,
             Builder builder) {
-        super(simpleName, mappedFieldType, multiFields, copyTo);
+        super(simpleName, mappedFieldType, multiFields, copyTo, builder.script.get() != null, builder.onScriptError.get());
         this.store = builder.store.getValue();
         this.indexed = builder.index.getValue();
         this.hasDocValues = builder.docValues.getValue();
@@ -628,11 +668,14 @@ public final class DateFieldMapper extends FieldMapper {
         this.resolution = resolution;
         this.ignoreMalformedByDefault = builder.ignoreMalformed.getDefaultValue();
         this.indexCreatedVersion = builder.indexCreatedVersion;
+        this.script = builder.script.get();
+        this.scriptCompiler = builder.scriptCompiler;
+        this.scriptValues = builder.scriptValues();
     }
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new Builder(simpleName(), resolution, null, ignoreMalformedByDefault, indexCreatedVersion).init(this);
+        return new Builder(simpleName(), resolution, null, scriptCompiler, ignoreMalformedByDefault, indexCreatedVersion).init(this);
     }
 
     @Override
@@ -678,6 +721,10 @@ public final class DateFieldMapper extends FieldMapper {
             }
         }
 
+        indexValue(context, timestamp);
+    }
+
+    private void indexValue(ParseContext context, long timestamp) {
         if (indexed) {
             context.doc().add(new LongPoint(fieldType().name(), timestamp));
         }
@@ -691,6 +738,11 @@ public final class DateFieldMapper extends FieldMapper {
         }
     }
 
+    @Override
+    protected void indexScriptValues(SearchLookup searchLookup, LeafReaderContext readerContext, int doc, ParseContext parseContext) {
+        this.scriptValues.valuesForDoc(searchLookup, readerContext, doc, v -> indexValue(parseContext, v));
+    }
+
     public boolean getIgnoreMalformed() {
         return ignoreMalformed;
     }

server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java

@@ -302,7 +302,7 @@ final class DynamicFieldsBuilder {
             Settings settings = context.indexSettings().getSettings();
             boolean ignoreMalformed = FieldMapper.IGNORE_MALFORMED_SETTING.get(settings);
             createDynamicField(new DateFieldMapper.Builder(name, DateFieldMapper.Resolution.MILLISECONDS,
-                dateTimeFormatter, ignoreMalformed, context.indexSettings().getIndexVersionCreated()), context);
+                dateTimeFormatter, ScriptCompiler.NONE, ignoreMalformed, context.indexSettings().getIndexVersionCreated()), context);
         }
 
         void newDynamicBinaryField(ParseContext context, String name) throws IOException {

server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java

@@ -58,6 +58,7 @@ import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.InvalidIndexNameException;
 import org.elasticsearch.indices.ShardLimitValidator;
 import org.elasticsearch.indices.TestIndexNameExpressionResolver;
+import org.elasticsearch.script.ScriptCompiler;
 import org.elasticsearch.test.ClusterServiceUtils;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.TestThreadPool;
@@ -558,9 +559,13 @@ public class MetadataRolloverServiceTests extends ESTestCase {
 
         ThreadPool testThreadPool = new TestThreadPool(getTestName());
         try {
-            DateFieldMapper dateFieldMapper
-                = new DateFieldMapper.Builder("@timestamp", DateFieldMapper.Resolution.MILLISECONDS, null, false, Version.CURRENT)
-                .build(new ContentPath());
+            DateFieldMapper dateFieldMapper = new DateFieldMapper.Builder(
+                "@timestamp",
+                DateFieldMapper.Resolution.MILLISECONDS,
+                null,
+                ScriptCompiler.NONE,
+                false,
+                Version.CURRENT).build(new ContentPath());
             MappedFieldType mockedTimestampFieldType = mock(MappedFieldType.class);
             when(mockedTimestampFieldType.name()).thenReturn("_data_stream_timestamp");
             MetadataFieldMapper mockedTimestampField = new MetadataFieldMapper(mockedTimestampFieldType) {
@@ -656,9 +661,13 @@ public class MetadataRolloverServiceTests extends ESTestCase {
 
         ThreadPool testThreadPool = new TestThreadPool(getTestName());
         try {
-            DateFieldMapper dateFieldMapper
-                = new DateFieldMapper.Builder("@timestamp", DateFieldMapper.Resolution.MILLISECONDS, null, false, Version.CURRENT)
-                .build(new ContentPath());
+            DateFieldMapper dateFieldMapper = new DateFieldMapper.Builder(
+                "@timestamp",
+                DateFieldMapper.Resolution.MILLISECONDS,
+                null,
+                ScriptCompiler.NONE,
+                false,
+                Version.CURRENT).build(new ContentPath());
             MappedFieldType mockedTimestampFieldType = mock(MappedFieldType.class);
             when(mockedTimestampFieldType.name()).thenReturn("_data_stream_timestamp");
             MetadataFieldMapper mockedTimestampField = new MetadataFieldMapper(mockedTimestampFieldType) {

server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java

@@ -558,4 +558,25 @@ public class DateFieldMapperTests extends MapperTestCase {
     private String randomDecimalNanos(long maxMillis) {
         return Long.toString(randomLongBetween(0, maxMillis)) + "." + between(0, 999999);
     }
+
+    public void testScriptAndPrecludedParameters() {
+        {
+            Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> {
+                b.field("type", "date");
+                b.field("script", "test");
+                b.field("null_value", 7);
+            })));
+            assertThat(e.getMessage(),
+                equalTo("Failed to parse mapping: Field [null_value] cannot be set in conjunction with field [script]"));
+        }
+        {
+            Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> {
+                b.field("type", "date");
+                b.field("script", "test");
+                b.field("ignore_malformed", "true");
+            })));
+            assertThat(e.getMessage(),
+                equalTo("Failed to parse mapping: Field [ignore_malformed] cannot be set in conjunction with field [script]"));
+        }
+    }
 }

server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java

@@ -166,7 +166,7 @@ public class DateFieldTypeTests extends FieldTypeTestCase {
         assertEquals(expected, ft.termQuery(date, context));
 
         MappedFieldType unsearchable = new DateFieldType("field", false, false, true, DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER,
-            Resolution.MILLISECONDS, null, Collections.emptyMap());
+            Resolution.MILLISECONDS, null, null, Collections.emptyMap());
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
                 () -> unsearchable.termQuery(date, context));
         assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage());
@@ -201,7 +201,7 @@ public class DateFieldTypeTests extends FieldTypeTestCase {
             ft.rangeQuery("now", instant2, true, true, null, null, null, context));
 
         MappedFieldType unsearchable = new DateFieldType("field", false, false, true, DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER,
-            Resolution.MILLISECONDS, null, Collections.emptyMap());
+            Resolution.MILLISECONDS, null, null, Collections.emptyMap());
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
                 () -> unsearchable.rangeQuery(date1, date2, true, true, null, null, null, context));
         assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage());
@@ -269,7 +269,7 @@ public class DateFieldTypeTests extends FieldTypeTestCase {
 
     private static DateFieldType fieldType(Resolution resolution, String format, String nullValue) {
         DateFormatter formatter = DateFormatter.forPattern(format);
-        return new DateFieldType("field", true, false, true, formatter, resolution, nullValue, Collections.emptyMap());
+        return new DateFieldType("field", true, false, true, formatter, resolution, nullValue, null, Collections.emptyMap());
     }
 
     public void testFetchSourceValue() throws IOException {

server/src/test/java/org/elasticsearch/index/mapper/DateScriptMapperTests.java

@@ -0,0 +1,96 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.LeafReaderContext;
+import org.elasticsearch.common.time.DateFormatter;
+import org.elasticsearch.script.DateFieldScript;
+import org.elasticsearch.search.lookup.SearchLookup;
+
+import java.util.Map;
+import java.util.function.Consumer;
+
+public class DateScriptMapperTests extends MapperScriptTestCase<DateFieldScript.Factory> {
+
+    private static DateFieldScript.Factory factory(Consumer<DateFieldScript> executor) {
+        return new DateFieldScript.Factory() {
+            @Override
+            public DateFieldScript.LeafFactory newFactory(
+                String fieldName,
+                Map<String, Object> params,
+                SearchLookup searchLookup,
+                DateFormatter formatter
+            ) {
+                return new DateFieldScript.LeafFactory() {
+                    @Override
+                    public DateFieldScript newInstance(LeafReaderContext ctx) {
+                        return new DateFieldScript(fieldName, params, searchLookup, formatter, ctx) {
+                            @Override
+                            public void execute() {
+                                executor.accept(this);
+                            }
+                        };
+                    }
+                };
+            }
+        };
+    }
+
+    @Override
+    protected String type() {
+        return "date";
+    }
+
+    @Override
+    protected DateFieldScript.Factory serializableScript() {
+        return factory(s -> {});
+    }
+
+    @Override
+    protected DateFieldScript.Factory errorThrowingScript() {
+        return factory(s -> {
+            throw new UnsupportedOperationException("Oops");
+        });
+    }
+
+    @Override
+    protected DateFieldScript.Factory singleValueScript() {
+        return factory(s -> s.emit(1516729294000L));
+    }
+
+    @Override
+    protected DateFieldScript.Factory multipleValuesScript() {
+        return factory(s -> {
+            s.emit(1516729294000L);
+            s.emit(1516729295000L);
+        });
+    }
+
+    @Override
+    protected void assertMultipleValues(IndexableField[] fields) {
+        assertEquals(4, fields.length);
+        assertEquals("LongPoint <field:1516729294000>", fields[0].toString());
+        assertEquals("docValuesType=SORTED_NUMERIC<field:1516729294000>", fields[1].toString());
+        assertEquals("LongPoint <field:1516729295000>", fields[2].toString());
+        assertEquals("docValuesType=SORTED_NUMERIC<field:1516729295000>", fields[3].toString());
+    }
+
+    @Override
+    protected void assertDocValuesDisabled(IndexableField[] fields) {
+        assertEquals(1, fields.length);
+        assertEquals("LongPoint <field:1516729294000>", fields[0].toString());
+    }
+
+    @Override
+    protected void assertIndexDisabled(IndexableField[] fields) {
+        assertEquals(1, fields.length);
+        assertEquals("docValuesType=SORTED_NUMERIC<field:1516729294000>", fields[0].toString());
+    }
+}

server/src/test/java/org/elasticsearch/search/aggregations/AggregatorBaseTests.java

@@ -91,7 +91,7 @@ public class AggregatorBaseTests extends MapperServiceTestCase {
         AggregationContext context
     ) {
         MappedFieldType ft = new DateFieldMapper.DateFieldType(fieldName, indexed, false, true,
-            DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, resolution, null, Collections.emptyMap());
+            DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, resolution, null, null, Collections.emptyMap());
         return ValuesSourceConfig.resolveFieldOnly(ft, context);
     }
 

server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java

@@ -322,6 +322,7 @@ public class FiltersAggregatorTests extends AggregatorTestCase {
             DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER,
             Resolution.MILLISECONDS,
             null,
+            null,
             Collections.emptyMap()
         );
         AggregationBuilder builder = new FiltersAggregationBuilder(
@@ -555,6 +556,7 @@ public class FiltersAggregatorTests extends AggregatorTestCase {
             DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER,
             Resolution.MILLISECONDS,
             null,
+            null,
             Collections.emptyMap()
         );
         MappedFieldType intFt = new NumberFieldMapper.NumberFieldType("int", NumberType.INTEGER);
@@ -629,6 +631,7 @@ public class FiltersAggregatorTests extends AggregatorTestCase {
             DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER,
             Resolution.MILLISECONDS,
             null,
+            null,
             Collections.emptyMap()
         );
         MappedFieldType intFt = new NumberFieldMapper.NumberFieldType("int", NumberType.INTEGER);
@@ -693,6 +696,7 @@ public class FiltersAggregatorTests extends AggregatorTestCase {
             DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER,
             Resolution.MILLISECONDS,
             null,
+            null,
             Collections.emptyMap()
         );
         MappedFieldType intFt = new NumberFieldMapper.NumberFieldType("int", NumberType.INTEGER);

server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTestCase.java

@@ -114,6 +114,7 @@ public abstract class DateHistogramAggregatorTestCase extends AggregatorTestCase
             formatter,
             useNanosecondResolution ? DateFieldMapper.Resolution.NANOSECONDS : DateFieldMapper.Resolution.MILLISECONDS,
             null,
+            null,
             Collections.emptyMap()
         );
     }

server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregatorTests.java

@@ -188,6 +188,7 @@ public class DateRangeAggregatorTests extends AggregatorTestCase {
                 DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER,
                 Resolution.MILLISECONDS,
                 null,
+                null,
                 Collections.emptyMap()
             )
         );
@@ -354,7 +355,7 @@ public class DateRangeAggregatorTests extends AggregatorTestCase {
                           Consumer<InternalRange<? extends InternalRange.Bucket, ? extends InternalRange>> verify,
                           DateFieldMapper.Resolution resolution) throws IOException {
         DateFieldMapper.DateFieldType fieldType = new DateFieldMapper.DateFieldType(DATE_FIELD_NAME, true, false, true,
-            DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, resolution, null, Collections.emptyMap());
+            DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, resolution, null, null, Collections.emptyMap());
         DateRangeAggregationBuilder aggregationBuilder = new DateRangeAggregationBuilder("test_range_agg");
         aggregationBuilder.field(DATE_FIELD_NAME);
         aggregationBuilder.addRange("2015-01-01", "2015-12-31");

server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java

@@ -155,6 +155,7 @@ public class RangeAggregatorTests extends AggregatorTestCase {
             DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER,
             Resolution.MILLISECONDS,
             null,
+            null,
             Collections.emptyMap()
         );
 
@@ -178,7 +179,7 @@ public class RangeAggregatorTests extends AggregatorTestCase {
 
     public void testDateFieldNanosecondResolution() throws IOException {
         DateFieldMapper.DateFieldType fieldType = new DateFieldMapper.DateFieldType(DATE_FIELD_NAME, true, false, true,
-            DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, DateFieldMapper.Resolution.NANOSECONDS, null, Collections.emptyMap());
+            DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, DateFieldMapper.Resolution.NANOSECONDS, null, null, Collections.emptyMap());
 
         // These values should work because aggs scale nanosecond up to millisecond always.
         long milli1 = ZonedDateTime.of(2015, 11, 13, 16, 14, 34, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
@@ -201,7 +202,7 @@ public class RangeAggregatorTests extends AggregatorTestCase {
 
     public void  testMissingDateWithDateNanosField() throws IOException {
         DateFieldMapper.DateFieldType fieldType = new DateFieldMapper.DateFieldType(DATE_FIELD_NAME, true, false, true,
-            DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, DateFieldMapper.Resolution.NANOSECONDS, null, Collections.emptyMap());
+            DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, DateFieldMapper.Resolution.NANOSECONDS, null, null, Collections.emptyMap());
 
         // These values should work because aggs scale nanosecond up to millisecond always.
         long milli1 = ZonedDateTime.of(2015, 11, 13, 16, 14, 34, 0, ZoneOffset.UTC).toInstant().toEpochMilli();

x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/multiterms/MultiTermsAggregatorTests.java

@@ -7,22 +7,6 @@
 
 package org.elasticsearch.xpack.analytics.multiterms;
 
-import static org.elasticsearch.xpack.analytics.multiterms.MultiTermsAggregationBuilderTests.randomTermConfig;
-import static org.hamcrest.Matchers.closeTo;
-import static org.hamcrest.Matchers.contains;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasSize;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.function.Consumer;
-import java.util.function.Function;
-
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FloatDocValuesField;
 import org.apache.lucene.document.LongPoint;
@@ -64,6 +48,22 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 import org.elasticsearch.search.lookup.LeafDocLookup;
 import org.elasticsearch.xpack.analytics.AnalyticsPlugin;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Consumer;
+import java.util.function.Function;
+
+import static org.elasticsearch.xpack.analytics.multiterms.MultiTermsAggregationBuilderTests.randomTermConfig;
+import static org.hamcrest.Matchers.closeTo;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+
 public class MultiTermsAggregatorTests extends AggregatorTestCase {
 
     /**
@@ -651,6 +651,7 @@ public class MultiTermsAggregatorTests extends AggregatorTestCase {
             DateFormatter.forPattern("strict_date"),
             DateFieldMapper.Resolution.MILLISECONDS,
             null,
+            null,
             Collections.emptyMap()
         );
     }

x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/RateAggregatorTests.java

@@ -767,6 +767,7 @@ public class RateAggregatorTests extends AggregatorTestCase {
             DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER,
             DateFieldMapper.Resolution.MILLISECONDS,
             null,
+            null,
             Collections.emptyMap()
         );
     }