Adds support for date_nanos in Rollup Metric and DateHistogram Configs (#59349)

Closes #44505.

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/RollupField.java

@@ -61,7 +61,8 @@ public class RollupField {
         NUMERIC_FIELD_MAPPER_TYPES = types;
     }
 
-    public static final String DATE_FIELD_MAPPER_TYPE = DateFieldMapper.CONTENT_TYPE;
+    public static final List<String> DATE_FIELD_MAPPER_TYPES = List.of(DateFieldMapper.CONTENT_TYPE,
+        DateFieldMapper.DATE_NANOS_CONTENT_TYPE);
 
     /**
      * Format to the appropriate Rollup field name convention

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/DateHistogramGroupConfig.java

@@ -21,6 +21,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
+import org.elasticsearch.xpack.core.rollup.RollupField;
 
 import java.io.IOException;
 import java.time.ZoneId;
@@ -314,24 +315,31 @@ public class DateHistogramGroupConfig implements Writeable, ToXContentObject {
 
     public void validateMappings(Map<String, Map<String, FieldCapabilities>> fieldCapsResponse,
                                                              ActionRequestValidationException validationException) {
-
         Map<String, FieldCapabilities> fieldCaps = fieldCapsResponse.get(field);
         if (fieldCaps != null && fieldCaps.isEmpty() == false) {
-            if (fieldCaps.containsKey("date") && fieldCaps.size() == 1) {
-                if (fieldCaps.get("date").isAggregatable()) {
-                    return;
-                } else {
-                    validationException.addValidationError("The field [" + field + "] must be aggregatable across all indices, " +
-                                    "but is not.");
+            boolean matchesDateType = false;
+            for (String dateType : RollupField.DATE_FIELD_MAPPER_TYPES) {
+                if (fieldCaps.containsKey(dateType) && fieldCaps.size() == 1) {
+                    matchesDateType |= true;
+                    if (fieldCaps.get(dateType).isAggregatable()) {
+                        return;
+                    } else {
+                        validationException.addValidationError("The field [" + field + "] must be aggregatable across all indices, " +
+                            "but is not.");
+                    }
                 }
-
-            } else {
-                validationException.addValidationError("The field referenced by a date_histo group must be a [date] type across all " +
-                        "indices in the index pattern.  Found: " + fieldCaps.keySet().toString() + " for field [" + field + "]");
             }
-        }
-        validationException.addValidationError("Could not find a [date] field with name [" + field + "] in any of the indices matching " +
+            if (matchesDateType == false) {
+                validationException.addValidationError("The field referenced by a date_histo group must be one of type [" +
+                    Strings.collectionToCommaDelimitedString(RollupField.DATE_FIELD_MAPPER_TYPES) + "] across all " +
+                    "indices in the index pattern.  Found: " + fieldCaps.keySet().toString() + " for field [" + field + "]");
+            }
+        } else {
+            validationException.addValidationError("Could not find one of [" +
+                Strings.collectionToCommaDelimitedString(RollupField.DATE_FIELD_MAPPER_TYPES) + "] fields with name [" +
+                field + "] in any of the indices matching " +
                 "the index pattern.");
+        }
     }
 
     @Override

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/MetricConfig.java

@@ -115,19 +115,21 @@ public class MetricConfig implements Writeable, ToXContentObject {
                 }
                 if (RollupField.NUMERIC_FIELD_MAPPER_TYPES.contains(key)) {
                     // nothing to do as all metrics are supported by SUPPORTED_NUMERIC_METRICS currently
-                } else if (RollupField.DATE_FIELD_MAPPER_TYPE.equals(key)) {
+                } else if (RollupField.DATE_FIELD_MAPPER_TYPES.contains(key)) {
                     if (RollupField.SUPPORTED_DATE_METRICS.containsAll(metrics) == false) {
                         validationException.addValidationError(
-                            buildSupportedMetricError("date", RollupField.SUPPORTED_DATE_METRICS));
+                            buildSupportedMetricError(key, RollupField.SUPPORTED_DATE_METRICS));
                     }
                 } else {
-                    validationException.addValidationError("The field referenced by a metric group must be a [numeric] or [date] type, " +
+                    validationException.addValidationError("The field referenced by a metric group must be a [numeric] or [" +
+                        Strings.collectionToCommaDelimitedString(RollupField.DATE_FIELD_MAPPER_TYPES) + "] type, " +
                         "but found " + fieldCaps.keySet().toString() + " for field [" + field + "]");
                 }
             });
         } else {
-            validationException.addValidationError("Could not find a [numeric] or [date] field with name [" + field + "] in any of the " +
-                    "indices matching the index pattern.");
+            validationException.addValidationError("Could not find a [numeric] or [" +
+                Strings.collectionToCommaDelimitedString(RollupField.DATE_FIELD_MAPPER_TYPES) +
+                "] field with name [" + field + "] in any of the " + "indices matching the index pattern.");
         }
     }
 

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/ConfigTestHelpers.java

@@ -19,6 +19,7 @@ import org.elasticsearch.xpack.core.rollup.job.TermsGroupConfig;
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Random;
@@ -29,6 +30,7 @@ import java.util.stream.IntStream;
 import static com.carrotsearch.randomizedtesting.generators.RandomNumbers.randomIntBetween;
 import static com.carrotsearch.randomizedtesting.generators.RandomPicks.randomFrom;
 import static com.carrotsearch.randomizedtesting.generators.RandomStrings.randomAsciiAlphanumOfLengthBetween;
+import static org.elasticsearch.test.ESTestCase.randomSubsetOf;
 import static org.elasticsearch.test.ESTestCase.randomZone;
 
 public class ConfigTestHelpers {
@@ -69,7 +71,10 @@ public class ConfigTestHelpers {
     }
 
     public static DateHistogramGroupConfig randomDateHistogramGroupConfig(final Random random) {
-        final String field = randomField(random);
+        return randomDateHistogramGroupConfigWithField(random, randomField(random));
+    }
+
+    public static DateHistogramGroupConfig randomDateHistogramGroupConfigWithField(final Random random, final String field) {
         final DateHistogramInterval delay = random.nextBoolean() ? randomInterval() : null;
         final String timezone = random.nextBoolean() ? randomZone().getId() : null;
         if (random.nextBoolean()) {
@@ -128,26 +133,11 @@ public class ConfigTestHelpers {
 
     public static MetricConfig randomMetricConfig(final Random random) {
         final String field = randomAsciiAlphanumOfLengthBetween(random, 15, 25);  // large names so we don't accidentally collide
-        final List<String> metrics = new ArrayList<>();
-        if (random.nextBoolean()) {
-            metrics.add("min");
-        }
-        if (random.nextBoolean()) {
-            metrics.add("max");
-        }
-        if (random.nextBoolean()) {
-            metrics.add("sum");
-        }
-        if (random.nextBoolean()) {
-            metrics.add("avg");
-        }
-        if (random.nextBoolean()) {
-            metrics.add("value_count");
-        }
-        if (metrics.size() == 0) {
-            metrics.add("min");
-        }
-        return new MetricConfig(field, Collections.unmodifiableList(metrics));
+        return randomMetricConfigWithFieldAndMetrics(random, field, RollupField.SUPPORTED_METRICS);
+    }
+
+    public static MetricConfig randomMetricConfigWithFieldAndMetrics(final Random random, String field, Collection<String> metrics) {
+        return new MetricConfig(field, Collections.unmodifiableList(randomSubsetOf(randomIntBetween(random, 1, metrics.size()), metrics)));
     }
 
     public static TermsGroupConfig randomTermsGroupConfig(final Random random) {

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/DateHistogramGroupConfigSerializingTests.java

@@ -68,8 +68,8 @@ public class DateHistogramGroupConfigSerializingTests extends AbstractSerializin
         DateHistogramGroupConfig config = new DateHistogramGroupConfig.CalendarInterval("my_field",
             new DateHistogramInterval("1d"), null, null);
         config.validateMappings(responseMap, e);
-        assertThat(e.validationErrors().get(0), equalTo("Could not find a [date] field with name [my_field] in any of the " +
-                "indices matching the index pattern."));
+        assertThat(e.validationErrors().get(0), equalTo("Could not find one of [date,date_nanos] fields with name [my_field] in " +
+            "any of the indices matching the index pattern."));
     }
 
     public void testValidateNomatchingField() {
@@ -83,8 +83,8 @@ public class DateHistogramGroupConfigSerializingTests extends AbstractSerializin
         DateHistogramGroupConfig config = new DateHistogramGroupConfig.CalendarInterval("my_field",
             new DateHistogramInterval("1d"), null, null);
         config.validateMappings(responseMap, e);
-        assertThat(e.validationErrors().get(0), equalTo("Could not find a [date] field with name [my_field] in any of the " +
-                "indices matching the index pattern."));
+        assertThat(e.validationErrors().get(0), equalTo("Could not find one of [date,date_nanos] fields with name [my_field] in " +
+            "any of the indices matching the index pattern."));
     }
 
     public void testValidateFieldWrongType() {
@@ -98,8 +98,8 @@ public class DateHistogramGroupConfigSerializingTests extends AbstractSerializin
         DateHistogramGroupConfig config = new DateHistogramGroupConfig.CalendarInterval("my_field",
             new DateHistogramInterval("1d"), null, null);
         config.validateMappings(responseMap, e);
-        assertThat(e.validationErrors().get(0), equalTo("The field referenced by a date_histo group must be a [date] type across all " +
-                "indices in the index pattern.  Found: [keyword] for field [my_field]"));
+        assertThat(e.validationErrors().get(0), equalTo("The field referenced by a date_histo group must be one of type " +
+            "[date,date_nanos] across all indices in the index pattern.  Found: [keyword] for field [my_field]"));
     }
 
     public void testValidateFieldMixtureTypes() {
@@ -116,8 +116,8 @@ public class DateHistogramGroupConfigSerializingTests extends AbstractSerializin
         DateHistogramGroupConfig config = new DateHistogramGroupConfig.CalendarInterval("my_field",
             new DateHistogramInterval("1d"), null, null);
         config.validateMappings(responseMap, e);
-        assertThat(e.validationErrors().get(0), equalTo("The field referenced by a date_histo group must be a [date] type across all " +
-                "indices in the index pattern.  Found: [date, keyword] for field [my_field]"));
+        assertThat(e.validationErrors().get(0), equalTo("The field referenced by a date_histo group must be one of type " +
+            "[date,date_nanos] across all indices in the index pattern.  Found: [date, keyword] for field [my_field]"));
     }
 
     public void testValidateFieldMatchingNotAggregatable() {

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/MetricConfigSerializingTests.java

@@ -14,10 +14,11 @@ import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers;
 import org.elasticsearch.xpack.core.rollup.RollupField;
 
 import java.io.IOException;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 
 import static java.util.Collections.singletonList;
 import static org.hamcrest.Matchers.equalTo;
@@ -49,11 +50,11 @@ public class MetricConfigSerializingTests extends AbstractSerializingTestCase<Me
 
         MetricConfig config = new MetricConfig("my_field", singletonList("max"));
         config.validateMappings(responseMap, e);
-        assertThat(e.validationErrors().get(0), equalTo("Could not find a [numeric] or [date] field with name [my_field] in any" +
-            " of the indices matching the index pattern."));
+        assertThat(e.validationErrors().get(0), equalTo("Could not find a [numeric] or [date,date_nanos] field with name [my_field] " +
+            "in any of the indices matching the index pattern."));
     }
 
-    public void testValidateNomatchingField() {
+    public void testValidateNoMatchingField() {
         ActionRequestValidationException e = new ActionRequestValidationException();
         Map<String, Map<String, FieldCapabilities>> responseMap = new HashMap<>();
 
@@ -63,8 +64,8 @@ public class MetricConfigSerializingTests extends AbstractSerializingTestCase<Me
 
         MetricConfig config = new MetricConfig("my_field", singletonList("max"));
         config.validateMappings(responseMap, e);
-        assertThat(e.validationErrors().get(0), equalTo("Could not find a [numeric] or [date] field with name [my_field] in any" +
-            " of the indices matching the index pattern."));
+        assertThat(e.validationErrors().get(0), equalTo("Could not find a [numeric] or [date,date_nanos] field with name [my_field] " +
+            "in any of the indices matching the index pattern."));
     }
 
     public void testValidateFieldWrongType() {
@@ -77,7 +78,7 @@ public class MetricConfigSerializingTests extends AbstractSerializingTestCase<Me
 
         MetricConfig config = new MetricConfig("my_field", singletonList("max"));
         config.validateMappings(responseMap, e);
-        assertThat("The field referenced by a metric group must be a [numeric] or [date] type," +
+        assertThat("The field referenced by a metric group must be a [numeric] or [date,date_nanos] type," +
             " but found [keyword] for field [my_field]", is(in(e.validationErrors())));
     }
 
@@ -95,90 +96,54 @@ public class MetricConfigSerializingTests extends AbstractSerializingTestCase<Me
         assertThat(e.validationErrors().get(0), equalTo("The field [my_field] must be aggregatable across all indices, but is not."));
     }
 
-    public void testValidateDateFieldUnsupportedMetric() {
-        ActionRequestValidationException e = new ActionRequestValidationException();
+    public void testValidateDateFieldsUnsupportedMetric() {
         Map<String, Map<String, FieldCapabilities>> responseMap = new HashMap<>();
 
-        // Have to mock fieldcaps because the ctor's aren't public...
-        FieldCapabilities fieldCaps = mock(FieldCapabilities.class);
-        when(fieldCaps.isAggregatable()).thenReturn(true);
-        responseMap.put("my_field", Collections.singletonMap("date", fieldCaps));
+        for (String mappingType : RollupField.DATE_FIELD_MAPPER_TYPES) {
+            // Have to mock fieldcaps because the ctor's aren't public...
+            FieldCapabilities fieldCaps = mock(FieldCapabilities.class);
+            when(fieldCaps.isAggregatable()).thenReturn(true);
+            responseMap.put("my_field", Collections.singletonMap(mappingType, fieldCaps));
+
+            Set<String> unsupportedMetrics = new HashSet<>(RollupField.SUPPORTED_METRICS);
+            unsupportedMetrics.removeAll(RollupField.SUPPORTED_DATE_METRICS);
+            for (String unsupportedMetric : unsupportedMetrics) {
+                MetricConfig config = new MetricConfig("my_field", Collections.singletonList(unsupportedMetric));
+                ActionRequestValidationException e = new ActionRequestValidationException();
+                config.validateMappings(responseMap, e);
+                assertThat(e.validationErrors().get(0), equalTo("Only the metrics " + RollupField.SUPPORTED_DATE_METRICS.toString() +
+                    " are supported for [" + mappingType + "] types, but unsupported metrics [" + unsupportedMetric +
+                    "] supplied for field [my_field]"));
+            }
+        }
 
-        MetricConfig config = new MetricConfig("my_field", Arrays.asList("avg", "max"));
-        config.validateMappings(responseMap, e);
-        assertThat(e.validationErrors().get(0), equalTo("Only the metrics " + RollupField.SUPPORTED_DATE_METRICS.toString() +
-            " are supported for [date] types, but unsupported metrics [avg] supplied for field [my_field]"));
     }
 
     public void testValidateMatchingField() {
         ActionRequestValidationException e = new ActionRequestValidationException();
         Map<String, Map<String, FieldCapabilities>> responseMap = new HashMap<>();
 
-        // Have to mock fieldcaps because the ctor's aren't public...
-        FieldCapabilities fieldCaps = mock(FieldCapabilities.class);
-        when(fieldCaps.isAggregatable()).thenReturn(true);
-        responseMap.put("my_field", Collections.singletonMap("long", fieldCaps));
-
-        MetricConfig config = new MetricConfig("my_field", singletonList("max"));
-        config.validateMappings(responseMap, e);
-        assertThat(e.validationErrors().size(), equalTo(0));
-
-
-        fieldCaps = mock(FieldCapabilities.class);
-        when(fieldCaps.isAggregatable()).thenReturn(true);
-        responseMap.put("my_field", Collections.singletonMap("double", fieldCaps));
-        config = new MetricConfig("my_field", singletonList("max"));
-        config.validateMappings(responseMap, e);
-        assertThat(e.validationErrors().size(), equalTo(0));
-
-        fieldCaps = mock(FieldCapabilities.class);
-        when(fieldCaps.isAggregatable()).thenReturn(true);
-        responseMap.put("my_field", Collections.singletonMap("float", fieldCaps));
-        config = new MetricConfig("my_field", singletonList("max"));
-        config.validateMappings(responseMap, e);
-        assertThat(e.validationErrors().size(), equalTo(0));
-
-        fieldCaps = mock(FieldCapabilities.class);
-        when(fieldCaps.isAggregatable()).thenReturn(true);
-        responseMap.put("my_field", Collections.singletonMap("short", fieldCaps));
-        config = new MetricConfig("my_field", singletonList("max"));
-        config.validateMappings(responseMap, e);
-        assertThat(e.validationErrors().size(), equalTo(0));
-
-        fieldCaps = mock(FieldCapabilities.class);
-        when(fieldCaps.isAggregatable()).thenReturn(true);
-        responseMap.put("my_field", Collections.singletonMap("byte", fieldCaps));
-        config = new MetricConfig("my_field", singletonList("max"));
-        config.validateMappings(responseMap, e);
-        assertThat(e.validationErrors().size(), equalTo(0));
-
-        fieldCaps = mock(FieldCapabilities.class);
-        when(fieldCaps.isAggregatable()).thenReturn(true);
-        responseMap.put("my_field", Collections.singletonMap("half_float", fieldCaps));
-        config = new MetricConfig("my_field", singletonList("max"));
-        config.validateMappings(responseMap, e);
-        assertThat(e.validationErrors().size(), equalTo(0));
-
-        fieldCaps = mock(FieldCapabilities.class);
-        when(fieldCaps.isAggregatable()).thenReturn(true);
-        responseMap.put("my_field", Collections.singletonMap("scaled_float", fieldCaps));
-        config = new MetricConfig("my_field", singletonList("max"));
-        config.validateMappings(responseMap, e);
-        assertThat(e.validationErrors().size(), equalTo(0));
-
-        fieldCaps = mock(FieldCapabilities.class);
-        when(fieldCaps.isAggregatable()).thenReturn(true);
-        responseMap.put("my_field", Collections.singletonMap("integer", fieldCaps));
-        config = new MetricConfig("my_field", singletonList("max"));
-        config.validateMappings(responseMap, e);
-        assertThat(e.validationErrors().size(), equalTo(0));
-
-        fieldCaps = mock(FieldCapabilities.class);
-        when(fieldCaps.isAggregatable()).thenReturn(true);
-        responseMap.put("my_field", Collections.singletonMap("date", fieldCaps));
-        config = new MetricConfig("my_field", singletonList("max"));
-        config.validateMappings(responseMap, e);
-        assertThat(e.validationErrors().size(), equalTo(0));
+        for (String numericType : RollupField.NUMERIC_FIELD_MAPPER_TYPES) {
+            // Have to mock fieldcaps because the ctor's aren't public...
+            FieldCapabilities fieldCaps = mock(FieldCapabilities.class);
+            when(fieldCaps.isAggregatable()).thenReturn(true);
+            responseMap.put("my_field", Collections.singletonMap(numericType, fieldCaps));
+            MetricConfig config = ConfigTestHelpers
+                .randomMetricConfigWithFieldAndMetrics(random(), "my_field", RollupField.SUPPORTED_NUMERIC_METRICS);
+            config.validateMappings(responseMap, e);
+            assertThat(e.validationErrors().size(), equalTo(0));
+        }
+
+        for (String dateType : RollupField.DATE_FIELD_MAPPER_TYPES) {
+            // Have to mock fieldcaps because the ctor's aren't public...
+            FieldCapabilities fieldCaps = mock(FieldCapabilities.class);
+            when(fieldCaps.isAggregatable()).thenReturn(true);
+            responseMap.put("my_field", Collections.singletonMap(dateType, fieldCaps));
+            MetricConfig config = ConfigTestHelpers
+                .randomMetricConfigWithFieldAndMetrics(random(), "my_field", RollupField.SUPPORTED_DATE_METRICS);
+            config.validateMappings(responseMap, e);
+            assertThat(e.validationErrors().size(), equalTo(0));
+        }
     }
 
 }

x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/put_job.yml

@@ -208,7 +208,7 @@ setup:
 "Validation failures":
 
   - do:
-      catch: /Could not find a \[numeric\] or \[date\] field with name \[field_doesnt_exist\] in any of the indices matching the index pattern/
+      catch: /Could not find a \[numeric\] or \[date,date_nanos\] field with name \[field_doesnt_exist\] in any of the indices matching the index pattern/
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       rollup.put_job: