[Transform] data nanos/date histogram IT (#53654)

add an integration test for date nanos in combination with date_histogram

x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java

@@ -56,6 +56,7 @@ public class TransformPivotRestIT extends TransformRestTestCase {
         }
 
         createReviewsIndex();
+        createReviewsIndexNano();
         indicesCreated = true;
         setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME);
 
@@ -496,9 +497,17 @@ public class TransformPivotRestIT extends TransformRestTestCase {
     }
 
     public void testDateHistogramPivot() throws Exception {
-        String transformId = "simple_date_histogram_pivot";
-        String transformIndex = "pivot_reviews_via_date_histogram";
-        setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, transformIndex);
+        assertDateHistogramPivot(REVIEWS_INDEX_NAME);
+    }
+
+    public void testDateHistogramPivotNanos() throws Exception {
+        assertDateHistogramPivot(REVIEWS_DATE_NANO_INDEX_NAME);
+    }
+
+    private void assertDateHistogramPivot(String indexName) throws Exception {
+        String transformId = "simple_date_histogram_pivot_" + indexName;
+        String transformIndex = "pivot_reviews_via_date_histogram_" + indexName;
+        setupDataAccessRole(DATA_ACCESS_ROLE, indexName, transformIndex);
 
         final Request createTransformRequest = createRequestWithAuth(
             "PUT",
@@ -506,13 +515,7 @@ public class TransformPivotRestIT extends TransformRestTestCase {
             BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS
         );
 
-        String config = "{"
-            + " \"source\": {\"index\":\""
-            + REVIEWS_INDEX_NAME
-            + "\"},"
-            + " \"dest\": {\"index\":\""
-            + transformIndex
-            + "\"},";
+        String config = "{" + " \"source\": {\"index\":\"" + indexName + "\"}," + " \"dest\": {\"index\":\"" + transformIndex + "\"},";
 
         config += " \"pivot\": {"
             + "   \"group_by\": {"

x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java

@@ -50,6 +50,7 @@ public abstract class TransformRestTestCase extends ESRestTestCase {
     private static final String BASIC_AUTH_VALUE_SUPER_USER = basicAuthHeaderValue("x_pack_rest_user", TEST_PASSWORD_SECURE_STRING);
 
     protected static final String REVIEWS_INDEX_NAME = "reviews";
+    protected static final String REVIEWS_DATE_NANO_INDEX_NAME = "reviews_nano";
 
     private static boolean useDeprecatedEndpoints;
 
@@ -75,18 +76,20 @@ public abstract class TransformRestTestCase extends ESRestTestCase {
         return super.buildClient(settings, hosts);
     }
 
-    protected void createReviewsIndex(String indexName, int numDocs) throws IOException {
+    protected void createReviewsIndex(String indexName, int numDocs, String dateType) throws IOException {
         int[] distributionTable = { 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 4, 4, 4, 3, 3, 2, 1, 1, 1 };
 
         // create mapping
         try (XContentBuilder builder = jsonBuilder()) {
             builder.startObject();
             {
-                builder.startObject("mappings")
-                    .startObject("properties")
-                    .startObject("timestamp")
-                    .field("type", "date")
-                    .endObject()
+                builder.startObject("mappings").startObject("properties").startObject("timestamp").field("type", dateType);
+
+                if (dateType.equals("date_nanos")) {
+                    builder.field("format", "strict_date_optional_time_nanos");
+                }
+
+                builder.endObject()
                     .startObject("user_id")
                     .field("type", "keyword")
                     .endObject()
@@ -128,7 +131,13 @@ public abstract class TransformRestTestCase extends ESRestTestCase {
             int sec = 10 + (i % 49);
             String location = (user + 10) + "," + (user + 15);
 
-            String date_string = "2017-01-" + day + "T" + hour + ":" + min + ":" + sec + "Z";
+            String date_string = "2017-01-" + day + "T" + hour + ":" + min + ":" + sec;
+            if (dateType.equals("date_nanos")) {
+                String randomNanos = "," + randomIntBetween(100000000, 999999999);
+                date_string += randomNanos;
+            }
+            date_string += "Z";
+
             bulk.append("{\"user_id\":\"")
                 .append("user_")
                 .append(user)
@@ -170,7 +179,7 @@ public abstract class TransformRestTestCase extends ESRestTestCase {
     }
 
     protected void createReviewsIndex(String indexName) throws IOException {
-        createReviewsIndex(indexName, 1000);
+        createReviewsIndex(indexName, 1000, "date");
     }
 
     protected void createPivotReviewsTransform(String transformId, String transformIndex, String query) throws IOException {
@@ -182,6 +191,10 @@ public abstract class TransformRestTestCase extends ESRestTestCase {
         createPivotReviewsTransform(transformId, transformIndex, query, pipeline, null);
     }
 
+    protected void createReviewsIndexNano() throws IOException {
+        createReviewsIndex(REVIEWS_DATE_NANO_INDEX_NAME, 1000, "date_nanos");
+    }
+
     protected void createContinuousPivotReviewsTransform(String transformId, String transformIndex, String authHeader) throws IOException {
 
         final Request createTransformRequest = createRequestWithAuth("PUT", getTransformEndpoint() + transformId, authHeader);

x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformTaskFailedStateIT.java

@@ -64,7 +64,7 @@ public class TransformTaskFailedStateIT extends TransformRestTestCase {
 
     public void testForceStopFailedTransform() throws Exception {
         String transformId = "test-force-stop-failed-transform";
-        createReviewsIndex(REVIEWS_INDEX_NAME, 10);
+        createReviewsIndex(REVIEWS_INDEX_NAME, 10, "date");
         String transformIndex = "failure_pivot_reviews";
         createDestinationIndexWithBadMapping(transformIndex);
         createContinuousPivotReviewsTransform(transformId, transformIndex, null);
@@ -102,7 +102,7 @@ public class TransformTaskFailedStateIT extends TransformRestTestCase {
 
     public void testStartFailedTransform() throws Exception {
         String transformId = "test-force-start-failed-transform";
-        createReviewsIndex(REVIEWS_INDEX_NAME, 10);
+        createReviewsIndex(REVIEWS_INDEX_NAME, 10, "date");
         String transformIndex = "failure_pivot_reviews";
         createDestinationIndexWithBadMapping(transformIndex);
         createContinuousPivotReviewsTransform(transformId, transformIndex, null);