Change default format for date_nanos field (#70463)

This commit updates the default format of date_nanos field
on existing and new indices to use `strict_date_optional_time_nanos` instead of
`strict_date_optional_time`.
Using `strict_date_optional_time` as the default format for date_nanos doesn't
make sense because it accepts and parses dates with nanosecond precision,
but when it formats it drops the nanoseconds.
The change should be transparent for users, these formats accept the same input.

Relates #69192
Closes #67063

docs/reference/mapping/types/date_nanos.asciidoc

@@ -18,15 +18,7 @@ back to a string depending on the date format that is associated with the field.
 Date formats can be customised, but if no `format` is specified then it uses
 the default:
 
-    "strict_date_optional_time||epoch_millis"
-
-This means that it will accept dates with optional timestamps, which conform
-to the formats supported by
-<<strict-date-time,`strict_date_optional_time`>> including up to nine second
-fractionals or milliseconds-since-the-epoch (thus losing precision on the
-nano second part). Using <<strict-date-time,`strict_date_optional_time`>> will 
-format the result up to only three second fractionals. To
-print and parse up to nine digits of resolution, use <<strict-date-time-nanos,`strict_date_optional_time_nanos`>>.
+    "strict_date_optional_time_nanos||epoch_millis"
 
 For instance:
 
@@ -74,7 +66,7 @@ GET my-index-000001/_search
   "docvalue_fields" : [
     {
       "field" : "date",
-      "format": "strict_date_time" <7>
+      "format": "strict_date_optional_time_nanos" <7>
     }
   ]
 }

qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml

@@ -256,3 +256,24 @@
   - do:
       indices.get_index_template:
         name: my-it
+
+---
+"Verify date_nanos default format":
+  - do:
+      indices.create:
+        index: index_nanos
+        body:
+          mappings:
+            properties:
+              date:
+                type: date
+              date_nanos:
+                type: date_nanos
+
+  - do:
+      index:
+        index: index_nanos
+        id: 0
+        body:
+          date: "2015-01-01T12:10:30.123456789Z"
+          date_nanos: "2015-01-01T12:10:30.123456789Z"

qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml

@@ -186,3 +186,16 @@
   - match: {component_templates.0.component_template.template.settings: {index: {number_of_shards: '1', number_of_replicas: '0'}}}
   - is_true: component_templates.0.component_template.template.mappings
   - match: {component_templates.0.component_template.template.aliases: {aliasname: {}}}
+
+---
+"Verify date_nanos default format":
+  - do:
+      search:
+        index: index_nanos
+        body:
+          fields: ["date_nanos", "date"]
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0._id: "0" }
+  - match: { hits.hits.0.fields.date: [ "2015-01-01T12:10:30.123Z" ] }
+  - match: { hits.hits.0.fields.date_nanos: [ "2015-01-01T12:10:30.123456789Z" ] }

server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java

@@ -64,6 +64,8 @@ public final class DateFieldMapper extends FieldMapper {
     public static final String CONTENT_TYPE = "date";
     public static final String DATE_NANOS_CONTENT_TYPE = "date_nanos";
     public static final DateFormatter DEFAULT_DATE_TIME_FORMATTER = DateFormatter.forPattern("strict_date_optional_time||epoch_millis");
+    public static final DateFormatter DEFAULT_DATE_TIME_NANOS_FORMATTER =
+        DateFormatter.forPattern("strict_date_optional_time_nanos||epoch_millis");
     private static final DateMathParser EPOCH_MILLIS_PARSER = DateFormatter.forPattern("epoch_millis").toDateMathParser();
 
     public enum Resolution {
@@ -216,8 +218,7 @@ public final class DateFieldMapper extends FieldMapper {
 
         private final Parameter<Map<String, String>> meta = Parameter.metaParam();
 
-        private final Parameter<String> format
-            = Parameter.stringParam("format", false, m -> toType(m).format, DEFAULT_DATE_TIME_FORMATTER.pattern());
+        private final Parameter<String> format;
         private final Parameter<Locale> locale = new Parameter<>("locale", false, () -> Locale.ROOT,
             (n, c, o) -> LocaleUtils.parse(o.toString()), m -> toType(m).locale);
 
@@ -235,6 +236,10 @@ public final class DateFieldMapper extends FieldMapper {
             this.indexCreatedVersion = indexCreatedVersion;
             this.ignoreMalformed
                 = Parameter.boolParam("ignore_malformed", true, m -> toType(m).ignoreMalformed, ignoreMalformedByDefault);
+
+            DateFormatter defaultFormat = resolution == Resolution.MILLISECONDS ?
+                DEFAULT_DATE_TIME_FORMATTER : DEFAULT_DATE_TIME_NANOS_FORMATTER;
+            this.format = Parameter.stringParam("format", false, m -> toType(m).format, defaultFormat.pattern());
             if (dateFormatter != null) {
                 this.format.setValue(dateFormatter.pattern());
                 this.locale.setValue(dateFormatter.locale());

server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java

@@ -319,6 +319,17 @@ public class DateFieldMapperTests extends MapperTestCase {
         assertEquals(List.of(date), fetchFromDocValues(mapperService, ft, format, 1589578382123L));
     }
 
+    public void testFormatPreserveNanos() throws IOException {
+        MapperService mapperService = createMapperService(
+            fieldMapping(b -> b.field("type", "date_nanos"))
+        );
+        DateFieldMapper.DateFieldType ft = (DateFieldMapper.DateFieldType) mapperService.fieldType("field");
+        assertEquals(ft.dateTimeFormatter, DateFieldMapper.DEFAULT_DATE_TIME_NANOS_FORMATTER);
+        DocValueFormat format = ft.docValueFormat(null, null);
+        String date = "2020-05-15T21:33:02.123456789Z";
+        assertEquals(List.of(date), fetchFromDocValues(mapperService, ft, format, date));
+    }
+
     public void testFetchDocValuesNanos() throws IOException {
         MapperService mapperService = createMapperService(
             fieldMapping(b -> b.field("type", "date_nanos").field("format", "strict_date_time||epoch_millis"))
@@ -494,7 +505,7 @@ public class DateFieldMapperTests extends MapperTestCase {
     }
 
     private String randomIs8601Nanos(long maxMillis) {
-        String date = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(randomLongBetween(0, maxMillis));
+        String date = DateFieldMapper.DEFAULT_DATE_TIME_NANOS_FORMATTER.formatMillis(randomLongBetween(0, maxMillis));
         date = date.substring(0, date.length() - 1);  // Strip off trailing "Z"
         return date + String.format(Locale.ROOT, "%06d", between(0, 999999)) + "Z";  // Add nanos and the "Z"
     }