[Transform] use ISO dates in output instead of epoch millis (#65584)

Transform writes dates as epoch millis, this does not work for historic data in some cases or is
unsupported. Dates should be written as such. With this PR transform starts writing dates in ISO
format, but as existing transform might rely on the format it provides backwards compatibility for
old jobs as well as a setting to write dates as epoch millis.

fixes #63787

client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/SettingsConfig.java

@@ -21,6 +21,7 @@ package org.elasticsearch.client.transform.transforms;
 
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -34,30 +35,43 @@ public class SettingsConfig implements ToXContentObject {
 
     private static final ParseField MAX_PAGE_SEARCH_SIZE = new ParseField("max_page_search_size");
     private static final ParseField DOCS_PER_SECOND = new ParseField("docs_per_second");
+    private static final ParseField DATES_AS_EPOCH_MILLIS = new ParseField("dates_as_epoch_millis");
     private static final int DEFAULT_MAX_PAGE_SEARCH_SIZE = -1;
     private static final float DEFAULT_DOCS_PER_SECOND = -1F;
 
+    // use an integer as we need to code 4 states: true, false, null (unchanged), default (defined server side)
+    private static final int DEFAULT_DATES_AS_EPOCH_MILLIS = -1;
+
     private final Integer maxPageSearchSize;
     private final Float docsPerSecond;
+    private final Integer datesAsEpochMillis;
 
     private static final ConstructingObjectParser<SettingsConfig, Void> PARSER = new ConstructingObjectParser<>(
         "settings_config",
         true,
-        args -> new SettingsConfig((Integer) args[0], (Float) args[1])
+        args -> new SettingsConfig((Integer) args[0], (Float) args[1], (Integer) args[2])
     );
 
     static {
         PARSER.declareIntOrNull(optionalConstructorArg(), DEFAULT_MAX_PAGE_SEARCH_SIZE, MAX_PAGE_SEARCH_SIZE);
         PARSER.declareFloatOrNull(optionalConstructorArg(), DEFAULT_DOCS_PER_SECOND, DOCS_PER_SECOND);
+        // this boolean requires 4 possible values: true, false, not_specified, default, therefore using a custom parser
+        PARSER.declareField(
+            optionalConstructorArg(),
+            p -> p.currentToken() == XContentParser.Token.VALUE_NULL ? DEFAULT_DATES_AS_EPOCH_MILLIS : p.booleanValue() ? 1 : 0,
+            DATES_AS_EPOCH_MILLIS,
+            ValueType.BOOLEAN_OR_NULL
+        );
     }
 
     public static SettingsConfig fromXContent(final XContentParser parser) {
         return PARSER.apply(parser, null);
     }
 
-    SettingsConfig(Integer maxPageSearchSize, Float docsPerSecond) {
+    SettingsConfig(Integer maxPageSearchSize, Float docsPerSecond, Integer datesAsEpochMillis) {
         this.maxPageSearchSize = maxPageSearchSize;
         this.docsPerSecond = docsPerSecond;
+        this.datesAsEpochMillis = datesAsEpochMillis;
     }
 
     @Override
@@ -77,6 +91,13 @@ public class SettingsConfig implements ToXContentObject {
                 builder.field(DOCS_PER_SECOND.getPreferredName(), docsPerSecond);
             }
         }
+        if (datesAsEpochMillis != null) {
+            if (datesAsEpochMillis.equals(DEFAULT_DATES_AS_EPOCH_MILLIS)) {
+                builder.field(DATES_AS_EPOCH_MILLIS.getPreferredName(), (Boolean) null);
+            } else {
+                builder.field(DATES_AS_EPOCH_MILLIS.getPreferredName(), datesAsEpochMillis > 0 ? true : false);
+            }
+        }
         builder.endObject();
         return builder;
     }
@@ -89,6 +110,10 @@ public class SettingsConfig implements ToXContentObject {
         return docsPerSecond;
     }
 
+    public Boolean getDatesAsEpochMillis() {
+        return datesAsEpochMillis != null ? datesAsEpochMillis > 0 : null;
+    }
+
     @Override
     public boolean equals(Object other) {
         if (other == this) {
@@ -99,12 +124,14 @@ public class SettingsConfig implements ToXContentObject {
         }
 
         SettingsConfig that = (SettingsConfig) other;
-        return Objects.equals(maxPageSearchSize, that.maxPageSearchSize) && Objects.equals(docsPerSecond, that.docsPerSecond);
+        return Objects.equals(maxPageSearchSize, that.maxPageSearchSize)
+            && Objects.equals(docsPerSecond, that.docsPerSecond)
+            && Objects.equals(datesAsEpochMillis, that.datesAsEpochMillis);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(maxPageSearchSize, docsPerSecond);
+        return Objects.hash(maxPageSearchSize, docsPerSecond, datesAsEpochMillis);
     }
 
     public static Builder builder() {
@@ -114,6 +141,7 @@ public class SettingsConfig implements ToXContentObject {
     public static class Builder {
         private Integer maxPageSearchSize;
         private Float docsPerSecond;
+        private Integer datesAsEpochMillis;
 
         /**
          * Sets the paging maximum paging maxPageSearchSize that transform can use when
@@ -143,8 +171,24 @@ public class SettingsConfig implements ToXContentObject {
             return this;
         }
 
+        /**
+         * Whether to write the output of a date aggregation as millis since epoch or as formatted string (ISO format).
+         *
+         * Transforms created before 7.11 write dates as epoch_millis. The new default is ISO string.
+         * You can use this setter to configure the old style writing as epoch millis.
+         *
+         * An explicit `null` resets to default.
+         *
+         * @param datesAsEpochMillis true if dates should be written as epoch_millis.
+         * @return the {@link Builder} with datesAsEpochMilli set.
+         */
+        public Builder setDatesAsEpochMillis(Boolean datesAsEpochMillis) {
+            this.datesAsEpochMillis = datesAsEpochMillis == null ? DEFAULT_DATES_AS_EPOCH_MILLIS : datesAsEpochMillis ? 1 : 0;
+            return this;
+        }
+
         public SettingsConfig build() {
-            return new SettingsConfig(maxPageSearchSize, docsPerSecond);
+            return new SettingsConfig(maxPageSearchSize, docsPerSecond, datesAsEpochMillis);
         }
     }
 }

client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/SettingsConfigTests.java

@@ -38,7 +38,11 @@ import static org.hamcrest.Matchers.equalTo;
 public class SettingsConfigTests extends AbstractXContentTestCase<SettingsConfig> {
 
     public static SettingsConfig randomSettingsConfig() {
-        return new SettingsConfig(randomBoolean() ? null : randomIntBetween(10, 10_000), randomBoolean() ? null : randomFloat());
+        return new SettingsConfig(
+            randomBoolean() ? null : randomIntBetween(10, 10_000),
+            randomBoolean() ? null : randomFloat(),
+            randomBoolean() ? null : randomIntBetween(-1, 1)
+        );
     }
 
     @Override
@@ -67,6 +71,7 @@ public class SettingsConfigTests extends AbstractXContentTestCase<SettingsConfig
 
         SettingsConfig emptyConfig = fromString("{}");
         assertNull(emptyConfig.getMaxPageSearchSize());
+        assertNull(emptyConfig.getDatesAsEpochMillis());
 
         settingsAsMap = xContentToMap(emptyConfig);
         assertTrue(settingsAsMap.isEmpty());
@@ -77,6 +82,15 @@ public class SettingsConfigTests extends AbstractXContentTestCase<SettingsConfig
         settingsAsMap = xContentToMap(config);
         assertThat(settingsAsMap.getOrDefault("max_page_search_size", "not_set"), equalTo("not_set"));
         assertNull(settingsAsMap.getOrDefault("docs_per_second", "not_set"));
+        assertThat(settingsAsMap.getOrDefault("dates_as_epoch_millis", "not_set"), equalTo("not_set"));
+
+        config = fromString("{\"dates_as_epoch_millis\" : null}");
+        assertFalse(config.getDatesAsEpochMillis());
+
+        settingsAsMap = xContentToMap(config);
+        assertThat(settingsAsMap.getOrDefault("max_page_search_size", "not_set"), equalTo("not_set"));
+        assertThat(settingsAsMap.getOrDefault("docs_per_second", "not_set"), equalTo("not_set"));
+        assertNull(settingsAsMap.getOrDefault("dates_as_epoch_millis", "not_set"));
     }
 
     public void testExplicitNullOnWriteBuilder() throws IOException {
@@ -87,9 +101,11 @@ public class SettingsConfigTests extends AbstractXContentTestCase<SettingsConfig
         Map<String, Object> settingsAsMap = xContentToMap(config);
         assertNull(settingsAsMap.getOrDefault("max_page_search_size", "not_set"));
         assertThat(settingsAsMap.getOrDefault("docs_per_second", "not_set"), equalTo("not_set"));
+        assertThat(settingsAsMap.getOrDefault("dates_as_epoch_millis", "not_set"), equalTo("not_set"));
 
         SettingsConfig emptyConfig = new SettingsConfig.Builder().build();
         assertNull(emptyConfig.getMaxPageSearchSize());
+        assertNull(emptyConfig.getDatesAsEpochMillis());
 
         settingsAsMap = xContentToMap(emptyConfig);
         assertTrue(settingsAsMap.isEmpty());
@@ -100,6 +116,16 @@ public class SettingsConfigTests extends AbstractXContentTestCase<SettingsConfig
         settingsAsMap = xContentToMap(config);
         assertThat(settingsAsMap.getOrDefault("max_page_search_size", "not_set"), equalTo("not_set"));
         assertNull(settingsAsMap.getOrDefault("docs_per_second", "not_set"));
+        assertThat(settingsAsMap.getOrDefault("dates_as_epoch_millis", "not_set"), equalTo("not_set"));
+
+        config = new SettingsConfig.Builder().setDatesAsEpochMillis(null).build();
+        // returns false, however it's `null` as in "use default", checked next
+        assertFalse(config.getDatesAsEpochMillis());
+
+        settingsAsMap = xContentToMap(config);
+        assertThat(settingsAsMap.getOrDefault("max_page_search_size", "not_set"), equalTo("not_set"));
+        assertThat(settingsAsMap.getOrDefault("docs_per_second", "not_set"), equalTo("not_set"));
+        assertNull(settingsAsMap.getOrDefault("dates_as_epoch_millis", "not_set"));
     }
 
     private Map<String, Object> xContentToMap(ToXContent xcontent) throws IOException {

client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/SettingsConfigTests.java

@@ -33,7 +33,8 @@ public class SettingsConfigTests extends AbstractResponseTestCase<
     public static org.elasticsearch.xpack.core.transform.transforms.SettingsConfig randomSettingsConfig() {
         return new org.elasticsearch.xpack.core.transform.transforms.SettingsConfig(
             randomBoolean() ? null : randomIntBetween(10, 10_000),
-            randomBoolean() ? null : randomFloat()
+            randomBoolean() ? null : randomFloat(),
+            randomBoolean() ? null : randomIntBetween(0, 1)
         );
     }
 
@@ -43,6 +44,7 @@ public class SettingsConfigTests extends AbstractResponseTestCase<
     ) {
         assertEquals(serverTestInstance.getMaxPageSearchSize(), clientInstance.getMaxPageSearchSize());
         assertEquals(serverTestInstance.getDocsPerSecond(), clientInstance.getDocsPerSecond());
+        assertEquals(serverTestInstance.getDatesAsEpochMillis(), clientInstance.getDatesAsEpochMillis());
     }
 
     @Override

docs/reference/rest-api/common-parms.asciidoc

@@ -148,10 +148,10 @@ The destination for the {transform}.
 end::dest[]
 
 tag::dest-index[]
-The _destination index_ for the {transform}. The mappings of the destination 
-index are deduced based on the source fields when possible. If alternate 
-mappings are required, use the 
-https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html[Create index API] 
+The _destination index_ for the {transform}. The mappings of the destination
+index are deduced based on the source fields when possible. If alternate
+mappings are required, use the
+https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html[Create index API]
 prior to starting the {transform}.
 end::dest-index[]
 
@@ -949,6 +949,13 @@ tag::transform-settings[]
 Defines optional {transform} settings.
 end::transform-settings[]
 
+tag::transform-settings-dates-as-epoch-milli[]
+Defines if dates in the ouput should be written as ISO formatted string (default)
+or as millis since epoch. `epoch_millis` has been the default for transforms created
+before version `7.11`. For compatible output set this to `true`.
+The default value is `false`.
+end::transform-settings-dates-as-epoch-milli[]
+
 tag::transform-settings-docs-per-second[]
 Specifies a limit on the number of input documents per second. This setting
 throttles the {transform} by adding a wait time between search requests. The

docs/reference/transform/apis/put-transform.asciidoc

@@ -18,7 +18,7 @@ Instantiates a {transform}.
 [[put-transform-prereqs]]
 == {api-prereq-title}
 
-If the {es} {security-features} are enabled, you must have the following 
+If the {es} {security-features} are enabled, you must have the following
 built-in roles and privileges:
 
 * `transform_admin`
@@ -134,6 +134,9 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings]
 .Properties of `settings`
 [%collapsible%open]
 ====
+`dates_as_epoch_millis`:::
+(Optional, boolean)
+include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-dates-as-epoch-milli]
 `docs_per_second`:::
 (Optional, float)
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-docs-per-second]
@@ -183,7 +186,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=sync-time]
 `delay`::::
 (Optional, <<time-units, time units>>)
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=sync-time-delay]
-    
+
 `field`::::
 (Required, string)
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=sync-time-field]

docs/reference/transform/apis/update-transform.asciidoc

@@ -18,7 +18,7 @@ Updates certain properties of a {transform}.
 [[update-transform-prereqs]]
 == {api-prereq-title}
 
-If the {es} {security-features} are enabled, you must have the following 
+If the {es} {security-features} are enabled, you must have the following
 built-in roles and privileges:
 
 * `transform_admin`
@@ -110,6 +110,9 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings]
 .Properties of `settings`
 [%collapsible%open]
 ====
+`dates_as_epoch_millis`:::
+(Optional, boolean)
+include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-dates-as-epoch-milli]
 `docs_per_second`:::
 (Optional, float)
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-docs-per-second]
@@ -131,7 +134,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source-transforms]
 `index`:::
 (Required, string or array)
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source-index-transforms]
-    
+
 `query`:::
 (Optional, object)
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source-query-transforms]

libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java

@@ -632,6 +632,7 @@ public final class ObjectParser<Value, Context> extends AbstractObjectParser<Val
         INT(VALUE_NUMBER, VALUE_STRING),
         INT_OR_NULL(VALUE_NUMBER, VALUE_STRING, VALUE_NULL),
         BOOLEAN(VALUE_BOOLEAN, VALUE_STRING),
+        BOOLEAN_OR_NULL(VALUE_BOOLEAN, VALUE_STRING, VALUE_NULL),
         STRING_ARRAY(START_ARRAY, VALUE_STRING),
         FLOAT_ARRAY(START_ARRAY, VALUE_NUMBER, VALUE_STRING),
         DOUBLE_ARRAY(START_ARRAY, VALUE_NUMBER, VALUE_STRING),

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java

@@ -35,6 +35,7 @@ public final class TransformField {
     public static final ParseField FORCE = new ParseField("force");
     public static final ParseField MAX_PAGE_SEARCH_SIZE = new ParseField("max_page_search_size");
     public static final ParseField DOCS_PER_SECOND = new ParseField("docs_per_second");
+    public static final ParseField DATES_AS_EPOCH_MILLIS = new ParseField("dates_as_epoch_millis");
     public static final ParseField FIELD = new ParseField("field");
     public static final ParseField SYNC = new ParseField("sync");
     public static final ParseField TIME_BASED_SYNC = new ParseField("time");

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java

@@ -6,11 +6,14 @@
 
 package org.elasticsearch.xpack.core.transform.transforms;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -28,33 +31,52 @@ public class SettingsConfig implements Writeable, ToXContentObject {
 
     private static final int DEFAULT_MAX_PAGE_SEARCH_SIZE = -1;
     private static final float DEFAULT_DOCS_PER_SECOND = -1F;
+    private static final int DEFAULT_DATES_AS_EPOCH_MILLIS = -1;
 
     private static ConstructingObjectParser<SettingsConfig, Void> createParser(boolean lenient) {
         ConstructingObjectParser<SettingsConfig, Void> parser = new ConstructingObjectParser<>(
             "transform_config_settings",
             lenient,
-            args -> new SettingsConfig((Integer) args[0], (Float) args[1])
+            args -> new SettingsConfig((Integer) args[0], (Float) args[1], (Integer) args[2])
         );
         parser.declareIntOrNull(optionalConstructorArg(), DEFAULT_MAX_PAGE_SEARCH_SIZE, TransformField.MAX_PAGE_SEARCH_SIZE);
         parser.declareFloatOrNull(optionalConstructorArg(), DEFAULT_DOCS_PER_SECOND, TransformField.DOCS_PER_SECOND);
+        // this boolean requires 4 possible values: true, false, not_specified, default, therefore using a custom parser
+        parser.declareField(
+            optionalConstructorArg(),
+            p -> p.currentToken() == XContentParser.Token.VALUE_NULL ? DEFAULT_DATES_AS_EPOCH_MILLIS : p.booleanValue() ? 1 : 0,
+            TransformField.DATES_AS_EPOCH_MILLIS,
+            ValueType.BOOLEAN_OR_NULL
+        );
         return parser;
     }
 
     private final Integer maxPageSearchSize;
     private final Float docsPerSecond;
+    private final Integer datesAsEpochMillis;
 
     public SettingsConfig() {
-        this(null, null);
+        this(null, null, (Integer) null);
+    }
+
+    public SettingsConfig(Integer maxPageSearchSize, Float docsPerSecond, Boolean datesAsEpochMillis) {
+        this(maxPageSearchSize, docsPerSecond, datesAsEpochMillis == null ? null : datesAsEpochMillis ? 1 : 0);
     }
 
-    public SettingsConfig(Integer maxPageSearchSize, Float docsPerSecond) {
+    public SettingsConfig(Integer maxPageSearchSize, Float docsPerSecond, Integer datesAsEpochMillis) {
         this.maxPageSearchSize = maxPageSearchSize;
         this.docsPerSecond = docsPerSecond;
+        this.datesAsEpochMillis = datesAsEpochMillis;
     }
 
     public SettingsConfig(final StreamInput in) throws IOException {
         this.maxPageSearchSize = in.readOptionalInt();
         this.docsPerSecond = in.readOptionalFloat();
+        if (in.getVersion().onOrAfter(Version.V_8_0_0)) { // todo: change to V_7_11
+            this.datesAsEpochMillis = in.readOptionalInt();
+        } else {
+            this.datesAsEpochMillis = DEFAULT_DATES_AS_EPOCH_MILLIS;
+        }
     }
 
     public Integer getMaxPageSearchSize() {
@@ -65,6 +87,14 @@ public class SettingsConfig implements Writeable, ToXContentObject {
         return docsPerSecond;
     }
 
+    public Boolean getDatesAsEpochMillis() {
+        return datesAsEpochMillis != null ? datesAsEpochMillis > 0 : null;
+    }
+
+    public Integer getDatesAsEpochMillisForUpdate() {
+        return datesAsEpochMillis;
+    }
+
     public ActionRequestValidationException validate(ActionRequestValidationException validationException) {
         // TODO: make this dependent on search.max_buckets
         if (maxPageSearchSize != null && (maxPageSearchSize < 10 || maxPageSearchSize > 10_000)) {
@@ -85,6 +115,9 @@ public class SettingsConfig implements Writeable, ToXContentObject {
     public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalInt(maxPageSearchSize);
         out.writeOptionalFloat(docsPerSecond);
+        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {  // todo: change to V_7_11_0
+            out.writeOptionalInt(datesAsEpochMillis);
+        }
     }
 
     @Override
@@ -97,6 +130,9 @@ public class SettingsConfig implements Writeable, ToXContentObject {
         if (docsPerSecond != null && (docsPerSecond.equals(DEFAULT_DOCS_PER_SECOND) == false)) {
             builder.field(TransformField.DOCS_PER_SECOND.getPreferredName(), docsPerSecond);
         }
+        if (datesAsEpochMillis != null && (datesAsEpochMillis.equals(DEFAULT_DATES_AS_EPOCH_MILLIS) == false)) {
+            builder.field(TransformField.DATES_AS_EPOCH_MILLIS.getPreferredName(), datesAsEpochMillis > 0 ? true : false);
+        }
         builder.endObject();
         return builder;
     }
@@ -111,12 +147,19 @@ public class SettingsConfig implements Writeable, ToXContentObject {
         }
 
         SettingsConfig that = (SettingsConfig) other;
-        return Objects.equals(maxPageSearchSize, that.maxPageSearchSize) && Objects.equals(docsPerSecond, that.docsPerSecond);
+        return Objects.equals(maxPageSearchSize, that.maxPageSearchSize)
+            && Objects.equals(docsPerSecond, that.docsPerSecond)
+            && Objects.equals(datesAsEpochMillis, that.datesAsEpochMillis);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(maxPageSearchSize, docsPerSecond);
+        return Objects.hash(maxPageSearchSize, docsPerSecond, datesAsEpochMillis);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this, true, true);
     }
 
     public static SettingsConfig fromXContent(final XContentParser parser, boolean lenient) throws IOException {
@@ -126,13 +169,12 @@ public class SettingsConfig implements Writeable, ToXContentObject {
     public static class Builder {
         private Integer maxPageSearchSize;
         private Float docsPerSecond;
+        private Integer datesAsEpochMillis;
 
         /**
          * Default builder
          */
-        public Builder() {
-
-        }
+        public Builder() {}
 
         /**
          * Builder starting from existing settings as base, for the purpose of partially updating settings.
@@ -142,6 +184,7 @@ public class SettingsConfig implements Writeable, ToXContentObject {
         public Builder(SettingsConfig base) {
             this.maxPageSearchSize = base.maxPageSearchSize;
             this.docsPerSecond = base.docsPerSecond;
+            this.datesAsEpochMillis = base.datesAsEpochMillis;
         }
 
         /**
@@ -172,6 +215,22 @@ public class SettingsConfig implements Writeable, ToXContentObject {
             return this;
         }
 
+        /**
+         * Whether to write the output of a date aggregation as millis since epoch or as formatted string (ISO format).
+         *
+         * Transforms created before 7.11 write dates as epoch_millis. The new default is ISO string.
+         * You can use this setter to configure the old style writing as epoch millis.
+         *
+         * An explicit `null` resets to default.
+         *
+         * @param datesAsEpochMillis true if dates should be written as epoch_millis.
+         * @return the {@link Builder} with datesAsEpochMilli set.
+         */
+        public Builder setDatesAsEpochMillis(Boolean datesAsEpochMillis) {
+            this.datesAsEpochMillis = datesAsEpochMillis == null ? DEFAULT_DATES_AS_EPOCH_MILLIS : datesAsEpochMillis ? 1 : 0;
+            return this;
+        }
+
         /**
          * Update settings according to given settings config.
          *
@@ -189,12 +248,17 @@ public class SettingsConfig implements Writeable, ToXContentObject {
                     ? null
                     : update.getMaxPageSearchSize();
             }
+            if (update.getDatesAsEpochMillisForUpdate() != null) {
+                this.datesAsEpochMillis = update.getDatesAsEpochMillisForUpdate().equals(DEFAULT_DATES_AS_EPOCH_MILLIS)
+                    ? null
+                    : update.getDatesAsEpochMillisForUpdate();
+            }
 
             return this;
         }
 
         public SettingsConfig build() {
-            return new SettingsConfig(maxPageSearchSize, docsPerSecond);
+            return new SettingsConfig(maxPageSearchSize, docsPerSecond, datesAsEpochMillis);
         }
     }
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java

@@ -344,8 +344,8 @@ public class TransformConfig extends AbstractDiffable<TransformConfig> implement
     public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
         final boolean excludeGenerated = params.paramAsBoolean(TransformField.EXCLUDE_GENERATED, false);
         final boolean forInternalStorage = params.paramAsBoolean(TransformField.FOR_INTERNAL_STORAGE, false);
-        assert (forInternalStorage && excludeGenerated) == false:
-            "unsupported behavior, exclude_generated is true and for_internal_storage is true";
+        assert (forInternalStorage
+            && excludeGenerated) == false : "unsupported behavior, exclude_generated is true and for_internal_storage is true";
         builder.startObject();
         builder.field(TransformField.ID.getPreferredName(), id);
         if (excludeGenerated == false) {
@@ -440,37 +440,63 @@ public class TransformConfig extends AbstractDiffable<TransformConfig> implement
     }
 
     /**
-     * Rewrites the transform config according to the latest format, for example moving deprecated
-     * settings to its new place.
+     * Rewrites the transform config according to the latest format.
+     *
+     * Operations cover:
+     *
+     *  - move deprecated settings to its new place
+     *  - change configuration options so it stays compatible (given a newer version)
      *
      * @param transformConfig original config
      * @return a rewritten transform config if a rewrite was necessary, otherwise the given transformConfig
      */
     public static TransformConfig rewriteForUpdate(final TransformConfig transformConfig) {
 
-        // quick checks for deprecated features, if none found just return the original
-        if (transformConfig.getPivotConfig() == null || transformConfig.getPivotConfig().getMaxPageSearchSize() == null) {
+        // quick check if a rewrite is required, if none found just return the original
+        // a failing quick check, does not mean a rewrite is necessary
+        if (transformConfig.getVersion() != null
+            && transformConfig.getVersion().onOrAfter(Version.V_8_0_0) // todo: V_7_11_0
+            && (transformConfig.getPivotConfig() == null || transformConfig.getPivotConfig().getMaxPageSearchSize() == null)) {
             return transformConfig;
         }
 
         Builder builder = new Builder(transformConfig);
 
-        if (transformConfig.getPivotConfig() != null && transformConfig.getPivotConfig().getMaxPageSearchSize() != null) {
-            // create a new pivot config but set maxPageSearchSize to null
-            PivotConfig newPivotConfig = new PivotConfig(
-                transformConfig.getPivotConfig().getGroupConfig(),
-                transformConfig.getPivotConfig().getAggregationConfig(),
-                null
-            );
-            builder.setPivotConfig(newPivotConfig);
+        // call apply rewrite without config, to only allow reading from the builder
+        return applyRewriteForUpdate(builder);
+    }
 
-            Integer maxPageSearchSizeDeprecated = transformConfig.getPivotConfig().getMaxPageSearchSize();
-            Integer maxPageSearchSize = transformConfig.getSettings().getMaxPageSearchSize() != null
-                ? transformConfig.getSettings().getMaxPageSearchSize()
+    private static TransformConfig applyRewriteForUpdate(Builder builder) {
+        // 1. Move pivot.max_page_size_search to settings.max_page_size_search
+        if (builder.getPivotConfig() != null && builder.getPivotConfig().getMaxPageSearchSize() != null) {
+
+            // find maxPageSearchSize value
+            Integer maxPageSearchSizeDeprecated = builder.getPivotConfig().getMaxPageSearchSize();
+            Integer maxPageSearchSize = builder.getSettings().getMaxPageSearchSize() != null
+                ? builder.getSettings().getMaxPageSearchSize()
                 : maxPageSearchSizeDeprecated;
 
-            builder.setSettings(new SettingsConfig(maxPageSearchSize, transformConfig.getSettings().getDocsPerSecond()));
+            // create a new pivot config but set maxPageSearchSize to null
+            builder.setPivotConfig(
+                new PivotConfig(builder.getPivotConfig().getGroupConfig(), builder.getPivotConfig().getAggregationConfig(), null)
+            );
+            // create new settings with maxPageSearchSize
+            builder.setSettings(
+                new SettingsConfig(
+                    maxPageSearchSize,
+                    builder.getSettings().getDocsPerSecond(),
+                    builder.getSettings().getDatesAsEpochMillis()
+                )
+            );
         }
+
+        // 2. set dates_as_epoch_millis to true for transforms < 7.11 to keep BWC
+        if (builder.getVersion() != null && builder.getVersion().before(Version.V_8_0_0)) { // todo: V_7_11_0
+            builder.setSettings(
+                new SettingsConfig(builder.getSettings().getMaxPageSearchSize(), builder.getSettings().getDocsPerSecond(), true)
+            );
+        }
+
         return builder.setVersion(Version.CURRENT).build();
     }
 
@@ -507,51 +533,91 @@ public class TransformConfig extends AbstractDiffable<TransformConfig> implement
             return this;
         }
 
+        String getId() {
+            return id;
+        }
+
         public Builder setSource(SourceConfig source) {
             this.source = source;
             return this;
         }
 
+        SourceConfig getSource() {
+            return source;
+        }
+
         public Builder setDest(DestConfig dest) {
             this.dest = dest;
             return this;
         }
 
+        DestConfig getDest() {
+            return dest;
+        }
+
         public Builder setFrequency(TimeValue frequency) {
             this.frequency = frequency;
             return this;
         }
 
+        TimeValue getFrequency() {
+            return frequency;
+        }
+
         public Builder setSyncConfig(SyncConfig syncConfig) {
             this.syncConfig = syncConfig;
             return this;
         }
 
+        SyncConfig getSyncConfig() {
+            return syncConfig;
+        }
+
         public Builder setDescription(String description) {
             this.description = description;
             return this;
         }
 
+        String getDescription() {
+            return description;
+        }
+
         public Builder setSettings(SettingsConfig settings) {
             this.settings = settings;
             return this;
         }
 
+        SettingsConfig getSettings() {
+            return settings;
+        }
+
         public Builder setHeaders(Map<String, String> headers) {
             this.headers = headers;
             return this;
         }
 
+        public Map<String, String> getHeaders() {
+            return headers;
+        }
+
         public Builder setPivotConfig(PivotConfig pivotConfig) {
             this.pivotConfig = pivotConfig;
             return this;
         }
 
+        PivotConfig getPivotConfig() {
+            return pivotConfig;
+        }
+
         Builder setVersion(Version version) {
             this.transformVersion = version;
             return this;
         }
 
+        Version getVersion() {
+            return transformVersion;
+        }
+
         public TransformConfig build() {
             return new TransformConfig(
                 id,

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfigTests.java

@@ -28,11 +28,15 @@ public class SettingsConfigTests extends AbstractSerializingTransformTestCase<Se
     private boolean lenient;
 
     public static SettingsConfig randomSettingsConfig() {
-        return new SettingsConfig(randomBoolean() ? null : randomIntBetween(10, 10_000), randomBoolean() ? null : randomFloat());
+        return new SettingsConfig(
+            randomBoolean() ? null : randomIntBetween(10, 10_000),
+            randomBoolean() ? null : randomFloat(),
+            randomBoolean() ? null : randomIntBetween(0, 1)
+        );
     }
 
     public static SettingsConfig randomNonEmptySettingsConfig() {
-        return new SettingsConfig(randomIntBetween(10, 10_000), randomFloat());
+        return new SettingsConfig(randomIntBetween(10, 10_000), randomFloat(), randomIntBetween(0, 1));
     }
 
     @Before
@@ -69,24 +73,30 @@ public class SettingsConfigTests extends AbstractSerializingTransformTestCase<Se
 
         assertThat(fromString("{\"docs_per_second\" : null}").getDocsPerSecond(), equalTo(-1F));
         assertNull(fromString("{}").getDocsPerSecond());
+
+        assertThat(fromString("{\"dates_as_epoch_millis\" : null}").getDatesAsEpochMillisForUpdate(), equalTo(-1));
+        assertNull(fromString("{}").getDatesAsEpochMillisForUpdate());
     }
 
     public void testUpdateUsingBuilder() throws IOException {
-        SettingsConfig config = fromString("{\"max_page_search_size\" : 10000, \"docs_per_second\" :42}");
+        SettingsConfig config = fromString("{\"max_page_search_size\" : 10000, \"docs_per_second\" :42, \"dates_as_epoch_millis\": true}");
 
         SettingsConfig.Builder builder = new SettingsConfig.Builder(config);
         builder.update(fromString("{\"max_page_search_size\" : 100}"));
 
         assertThat(builder.build().getMaxPageSearchSize(), equalTo(100));
         assertThat(builder.build().getDocsPerSecond(), equalTo(42F));
+        assertThat(builder.build().getDatesAsEpochMillisForUpdate(), equalTo(1));
 
         builder.update(fromString("{\"max_page_search_size\" : null}"));
         assertNull(builder.build().getMaxPageSearchSize());
         assertThat(builder.build().getDocsPerSecond(), equalTo(42F));
+        assertThat(builder.build().getDatesAsEpochMillisForUpdate(), equalTo(1));
 
-        builder.update(fromString("{\"max_page_search_size\" : 77, \"docs_per_second\" :null}"));
+        builder.update(fromString("{\"max_page_search_size\" : 77, \"docs_per_second\" :null, \"dates_as_epoch_millis\": null}"));
         assertThat(builder.build().getMaxPageSearchSize(), equalTo(77));
         assertNull(builder.build().getDocsPerSecond());
+        assertNull(builder.build().getDatesAsEpochMillisForUpdate());
     }
 
     public void testOmmitDefaultsOnWriteParser() throws IOException {
@@ -108,6 +118,12 @@ public class SettingsConfigTests extends AbstractSerializingTransformTestCase<Se
 
         settingsAsMap = xContentToMap(config);
         assertTrue(settingsAsMap.isEmpty());
+
+        config = fromString("{\"dates_as_epoch_millis\" : null}");
+        assertThat(config.getDatesAsEpochMillisForUpdate(), equalTo(-1));
+
+        settingsAsMap = xContentToMap(config);
+        assertTrue(settingsAsMap.isEmpty());
     }
 
     public void testOmmitDefaultsOnWriteBuilder() throws IOException {
@@ -129,6 +145,12 @@ public class SettingsConfigTests extends AbstractSerializingTransformTestCase<Se
 
         settingsAsMap = xContentToMap(config);
         assertTrue(settingsAsMap.isEmpty());
+
+        config = new SettingsConfig.Builder().setDatesAsEpochMillis(null).build();
+        assertThat(config.getDatesAsEpochMillisForUpdate(), equalTo(-1));
+
+        settingsAsMap = xContentToMap(config);
+        assertTrue(settingsAsMap.isEmpty());
     }
 
     private Map<String, Object> xContentToMap(ToXContent xcontent) throws IOException {

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java

@@ -350,11 +350,13 @@ public class TransformConfigTests extends AbstractSerializingTransformTestCase<T
         assertNull(transformConfigRewritten.getPivotConfig().getMaxPageSearchSize());
         assertNotNull(transformConfigRewritten.getSettings().getMaxPageSearchSize());
         assertEquals(111L, transformConfigRewritten.getSettings().getMaxPageSearchSize().longValue());
+        assertTrue(transformConfigRewritten.getSettings().getDatesAsEpochMillis());
+
         assertWarnings("[max_page_search_size] is deprecated inside pivot please use settings instead");
         assertEquals(Version.CURRENT, transformConfigRewritten.getVersion());
     }
 
-    public void testRewriteForUpdateConflicting() throws IOException {
+    public void testRewriteForUpdateMaxPageSizeSearchConflicting() throws IOException {
         String pivotTransform = "{"
             + " \"id\" : \"body_id\","
             + " \"source\" : {\"index\":\"src\"},"
@@ -389,6 +391,44 @@ public class TransformConfigTests extends AbstractSerializingTransformTestCase<T
         assertWarnings("[max_page_search_size] is deprecated inside pivot please use settings instead");
     }
 
+    public void testRewriteForBWCOfDateNormalization() throws IOException {
+        String pivotTransform = "{"
+            + " \"id\" : \"body_id\","
+            + " \"source\" : {\"index\":\"src\"},"
+            + " \"dest\" : {\"index\": \"dest\"},"
+            + " \"pivot\" : {"
+            + " \"group_by\": {"
+            + "   \"id\": {"
+            + "     \"terms\": {"
+            + "       \"field\": \"id\""
+            + "} } },"
+            + " \"aggs\": {"
+            + "   \"avg\": {"
+            + "     \"avg\": {"
+            + "       \"field\": \"points\""
+            + "} } }"
+            + "},"
+            + " \"version\" : \""
+            + Version.V_7_6_0.toString()
+            + "\""
+            + "}";
+
+        TransformConfig transformConfig = createTransformConfigFromString(pivotTransform, "body_id", true);
+        TransformConfig transformConfigRewritten = TransformConfig.rewriteForUpdate(transformConfig);
+
+        assertTrue(transformConfigRewritten.getSettings().getDatesAsEpochMillis());
+        assertEquals(Version.CURRENT, transformConfigRewritten.getVersion());
+
+        TransformConfig explicitTrueAfter711 = new TransformConfig.Builder(transformConfig).setSettings(
+            new SettingsConfig.Builder(transformConfigRewritten.getSettings()).setDatesAsEpochMillis(true).build()
+        ).setVersion(Version.V_8_0_0).build(); // todo: V_7_11_0
+
+        transformConfigRewritten = TransformConfig.rewriteForUpdate(explicitTrueAfter711);
+
+        assertTrue(transformConfigRewritten.getSettings().getDatesAsEpochMillis());
+        assertEquals(Version.V_8_0_0, transformConfigRewritten.getVersion()); // todo: V_7_11_0
+    }
+
     private TransformConfig createTransformConfigFromString(String json, String id) throws IOException {
         return createTransformConfigFromString(json, id, false);
     }

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdateTests.java

@@ -104,7 +104,7 @@ public class TransformConfigUpdateTests extends AbstractWireSerializingTransform
         TimeValue frequency = TimeValue.timeValueSeconds(10);
         SyncConfig syncConfig = new TimeSyncConfig("time_field", TimeValue.timeValueSeconds(30));
         String newDescription = "new description";
-        SettingsConfig settings = new SettingsConfig(4_000, 4_000.400F);
+        SettingsConfig settings = new SettingsConfig(4_000, 4_000.400F, true);
         update = new TransformConfigUpdate(sourceConfig, destConfig, frequency, syncConfig, newDescription, settings);
 
         Map<String, String> headers = Collections.singletonMap("foo", "bar");
@@ -136,7 +136,14 @@ public class TransformConfigUpdateTests extends AbstractWireSerializingTransform
             randomBoolean() ? null : Version.V_7_2_0.toString()
         );
 
-        TransformConfigUpdate update = new TransformConfigUpdate(null, null, null, null, null, new SettingsConfig(4_000, null));
+        TransformConfigUpdate update = new TransformConfigUpdate(
+            null,
+            null,
+            null,
+            null,
+            null,
+            new SettingsConfig(4_000, null, (Boolean) null)
+        );
         TransformConfig updatedConfig = update.apply(config);
 
         // for settings we allow partial updates, so changing 1 setting should not overwrite the other
@@ -144,18 +151,18 @@ public class TransformConfigUpdateTests extends AbstractWireSerializingTransform
         assertThat(updatedConfig.getSettings().getMaxPageSearchSize(), equalTo(4_000));
         assertThat(updatedConfig.getSettings().getDocsPerSecond(), equalTo(config.getSettings().getDocsPerSecond()));
 
-        update = new TransformConfigUpdate(null, null, null, null, null, new SettingsConfig(null, 43.244F));
+        update = new TransformConfigUpdate(null, null, null, null, null, new SettingsConfig(null, 43.244F, (Boolean) null));
         updatedConfig = update.apply(updatedConfig);
         assertThat(updatedConfig.getSettings().getMaxPageSearchSize(), equalTo(4_000));
         assertThat(updatedConfig.getSettings().getDocsPerSecond(), equalTo(43.244F));
 
         // now reset to default using the magic -1
-        update = new TransformConfigUpdate(null, null, null, null, null, new SettingsConfig(-1, null));
+        update = new TransformConfigUpdate(null, null, null, null, null, new SettingsConfig(-1, null, (Boolean) null));
         updatedConfig = update.apply(updatedConfig);
         assertNull(updatedConfig.getSettings().getMaxPageSearchSize());
         assertThat(updatedConfig.getSettings().getDocsPerSecond(), equalTo(43.244F));
 
-        update = new TransformConfigUpdate(null, null, null, null, null, new SettingsConfig(-1, -1F));
+        update = new TransformConfigUpdate(null, null, null, null, null, new SettingsConfig(-1, -1F, (Boolean) null));
         updatedConfig = update.apply(updatedConfig);
         assertNull(updatedConfig.getSettings().getMaxPageSearchSize());
         assertNull(updatedConfig.getSettings().getDocsPerSecond());

x-pack/plugin/src/test/resources/rest-api-spec/test/transform/preview_transforms.yml

@@ -84,17 +84,17 @@ setup:
             }
           }
   - match: { preview.0.airline: foo }
-  - match: { preview.0.by-hour: 1487376000000 }
+  - match: { preview.0.by-hour: 2017-02-18T00:00:00.000Z }
   - match: { preview.0.avg_response: 1.0 }
   - match: { preview.0.time.max: "2017-02-18T00:30:00.000Z" }
   - match: { preview.0.time.min: "2017-02-18T00:00:00.000Z" }
   - match: { preview.1.airline: bar }
-  - match: { preview.1.by-hour: 1487379600000 }
+  - match: { preview.1.by-hour: 2017-02-18T01:00:00.000Z }
   - match: { preview.1.avg_response: 42.0 }
   - match: { preview.1.time.max: "2017-02-18T01:00:00.000Z" }
   - match: { preview.1.time.min: "2017-02-18T01:00:00.000Z" }
   - match: { preview.2.airline: foo }
-  - match: { preview.2.by-hour: 1487379600000 }
+  - match: { preview.2.by-hour: 2017-02-18T01:00:00.000Z }
   - match: { preview.2.avg_response: 42.0 }
   - match: { preview.2.time.max: "2017-02-18T01:01:00.000Z" }
   - match: { preview.2.time.min: "2017-02-18T01:01:00.000Z" }
@@ -135,15 +135,15 @@ setup:
             }
           }
   - match: { preview.0.airline: foo }
-  - match: { preview.0.by-hour: 1487376000000 }
+  - match: { preview.0.by-hour: 2017-02-18T00:00:00.000Z }
   - match: { preview.0.avg_response: 1.0 }
   - match: { preview.0.my_field: 42 }
   - match: { preview.1.airline: bar }
-  - match: { preview.1.by-hour: 1487379600000 }
+  - match: { preview.1.by-hour: 2017-02-18T01:00:00.000Z }
   - match: { preview.1.avg_response: 42.0 }
   - match: { preview.1.my_field: 42 }
   - match: { preview.2.airline: foo }
-  - match: { preview.2.by-hour: 1487379600000 }
+  - match: { preview.2.by-hour: 2017-02-18T01:00:00.000Z }
   - match: { preview.2.avg_response: 42.0 }
   - match: { preview.2.my_field: 42 }
   - match: { generated_dest_index.mappings.properties.airline.type: "keyword" }

x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformProgressIT.java

@@ -6,6 +6,7 @@
 
 package org.elasticsearch.xpack.transform.integration;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.LatchedActionListener;
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
@@ -29,6 +30,7 @@ import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.xpack.core.transform.transforms.DestConfig;
+import org.elasticsearch.xpack.core.transform.transforms.SettingsConfig;
 import org.elasticsearch.xpack.core.transform.transforms.SourceConfig;
 import org.elasticsearch.xpack.core.transform.transforms.TransformConfig;
 import org.elasticsearch.xpack.core.transform.transforms.TransformProgress;
@@ -151,7 +153,7 @@ public class TransformProgressIT extends ESRestTestCase {
         PivotConfig pivotConfig = new PivotConfig(histgramGroupConfig, aggregationConfig, null);
         TransformConfig config = new TransformConfig(transformId, sourceConfig, destConfig, null, null, null, pivotConfig, null, null);
 
-        Pivot pivot = new Pivot(pivotConfig, transformId);
+        Pivot pivot = new Pivot(pivotConfig, transformId, new SettingsConfig(), Version.CURRENT);
 
         TransformProgress progress = getProgress(pivot, getProgressQuery(pivot, config.getSource().getIndex(), null));
 
@@ -179,7 +181,7 @@ public class TransformProgressIT extends ESRestTestCase {
             Collections.singletonMap("every_50", new HistogramGroupSource("missing_field", null, missingBucket, 50.0))
         );
         pivotConfig = new PivotConfig(histgramGroupConfig, aggregationConfig, null);
-        pivot = new Pivot(pivotConfig, transformId);
+        pivot = new Pivot(pivotConfig, transformId, new SettingsConfig(), Version.CURRENT);
 
         progress = getProgress(
             pivot,

x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByIT.java

@@ -10,6 +10,7 @@ import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.transform.transforms.DestConfig;
+import org.elasticsearch.client.transform.transforms.SettingsConfig;
 import org.elasticsearch.client.transform.transforms.SourceConfig;
 import org.elasticsearch.client.transform.transforms.TransformConfig;
 import org.elasticsearch.client.transform.transforms.pivot.DateHistogramGroupSource;
@@ -28,7 +29,6 @@ import org.elasticsearch.search.builder.SearchSourceBuilder;
 import java.io.IOException;
 import java.time.Instant;
 import java.time.ZoneId;
-import java.time.ZonedDateTime;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -43,15 +43,21 @@ public class DateHistogramGroupByIT extends ContinuousTestCase {
         .format(Instant.ofEpochMilli(42));
 
     private final boolean missing;
+    private final boolean datesAsEpochMillis;
 
     public DateHistogramGroupByIT() {
         missing = randomBoolean();
+        datesAsEpochMillis = randomBoolean();
     }
 
     @Override
     public TransformConfig createConfig() {
         TransformConfig.Builder transformConfigBuilder = new TransformConfig.Builder();
         addCommonBuilderParameters(transformConfigBuilder);
+        if (datesAsEpochMillis) {
+            transformConfigBuilder.setSettings(addCommonSetings(new SettingsConfig.Builder()).setDatesAsEpochMillis(true).build());
+        }
+
         transformConfigBuilder.setSource(new SourceConfig(CONTINUOUS_EVENTS_SOURCE_INDEX));
         transformConfigBuilder.setDest(new DestConfig(NAME, INGEST_PIPELINE));
         transformConfigBuilder.setId(NAME);
@@ -110,9 +116,16 @@ public class DateHistogramGroupByIT extends ContinuousTestCase {
             SearchHit searchHit = destIterator.next();
             Map<String, Object> source = searchHit.getSourceAsMap();
 
-            Long transformBucketKey = (Long) XContentMapValues.extractValue("second", source);
+            String transformBucketKey;
+            if (datesAsEpochMillis) {
+                transformBucketKey = ContinuousTestCase.STRICT_DATE_OPTIONAL_TIME_PRINTER_NANOS.withZone(ZoneId.of("UTC"))
+                    .format(Instant.ofEpochMilli((Long) XContentMapValues.extractValue("second", source)));
+            } else {
+                transformBucketKey = (String) XContentMapValues.extractValue("second", source);
+            }
+
             if (transformBucketKey == null) {
-                transformBucketKey = 42L;
+                transformBucketKey = MISSING_BUCKET_KEY;
             }
 
             // aggs return buckets with 0 doc_count while composite aggs skip over them
@@ -120,13 +133,12 @@ public class DateHistogramGroupByIT extends ContinuousTestCase {
                 assertTrue(sourceIterator.hasNext());
                 bucket = sourceIterator.next();
             }
-            long bucketKey = ((ZonedDateTime) bucket.getKey()).toEpochSecond() * 1000;
 
             // test correctness, the results from the aggregation and the results from the transform should be the same
             assertThat(
-                "Buckets did not match, source: " + source + ", expected: " + bucketKey + ", iteration: " + iteration,
+                "Buckets did not match, source: " + source + ", expected: " + bucket.getKeyAsString() + ", iteration: " + iteration,
                 transformBucketKey,
-                equalTo(bucketKey)
+                equalTo(bucket.getKeyAsString())
             );
             assertThat(
                 "Doc count did not match, source: " + source + ", expected: " + bucket.getDocCount() + ", iteration: " + iteration,

x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByOtherTimeFieldIT.java

@@ -4,6 +4,7 @@ import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.transform.transforms.DestConfig;
+import org.elasticsearch.client.transform.transforms.SettingsConfig;
 import org.elasticsearch.client.transform.transforms.SourceConfig;
 import org.elasticsearch.client.transform.transforms.TransformConfig;
 import org.elasticsearch.client.transform.transforms.pivot.DateHistogramGroupSource;
@@ -23,7 +24,8 @@ import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilde
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 
 import java.io.IOException;
-import java.time.ZonedDateTime;
+import java.time.Instant;
+import java.time.ZoneId;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -41,15 +43,20 @@ public class DateHistogramGroupByOtherTimeFieldIT extends ContinuousTestCase {
     private static final String NAME = "continuous-date-histogram-pivot-other-timefield-test";
 
     private final boolean addGroupByTerms;
+    private final boolean datesAsEpochMillis;
 
     public DateHistogramGroupByOtherTimeFieldIT() {
         addGroupByTerms = randomBoolean();
+        datesAsEpochMillis = randomBoolean();
     }
 
     @Override
     public TransformConfig createConfig() {
         TransformConfig.Builder transformConfigBuilder = new TransformConfig.Builder();
         addCommonBuilderParameters(transformConfigBuilder);
+        if (datesAsEpochMillis) {
+            transformConfigBuilder.setSettings(addCommonSetings(new SettingsConfig.Builder()).setDatesAsEpochMillis(true).build());
+        }
         transformConfigBuilder.setSource(new SourceConfig(CONTINUOUS_EVENTS_SOURCE_INDEX));
         transformConfigBuilder.setDest(new DestConfig(NAME, INGEST_PIPELINE));
         transformConfigBuilder.setId(NAME);
@@ -121,20 +128,25 @@ public class DateHistogramGroupByOtherTimeFieldIT extends ContinuousTestCase {
             SearchHit searchHit = destIterator.next();
             Map<String, Object> source = searchHit.getSourceAsMap();
 
-            Long transformBucketKey = (Long) XContentMapValues.extractValue("second", source);
+            String transformBucketKey;
+            if (datesAsEpochMillis) {
+                transformBucketKey = ContinuousTestCase.STRICT_DATE_OPTIONAL_TIME_PRINTER_NANOS.withZone(ZoneId.of("UTC"))
+                    .format(Instant.ofEpochMilli((Long) XContentMapValues.extractValue("second", source)));
+            } else {
+                transformBucketKey = (String) XContentMapValues.extractValue("second", source);
+            }
 
             // aggs return buckets with 0 doc_count while composite aggs skip over them
             while (bucket.getDocCount() == 0L) {
                 assertTrue(sourceIterator.hasNext());
                 bucket = sourceIterator.next();
             }
-            long bucketKey = ((ZonedDateTime) bucket.getKey()).toEpochSecond() * 1000;
 
             // test correctness, the results from the aggregation and the results from the transform should be the same
             assertThat(
-                "Buckets did not match, source: " + source + ", expected: " + bucketKey + ", iteration: " + iteration,
+                "Buckets did not match, source: " + source + ", expected: " + bucket.getKeyAsString() + ", iteration: " + iteration,
                 transformBucketKey,
-                equalTo(bucketKey)
+                equalTo(bucket.getKeyAsString())
             );
             assertThat(
                 "Doc count did not match, source: " + source + ", expected: " + bucket.getDocCount() + ", iteration: " + iteration,
@@ -172,10 +184,9 @@ public class DateHistogramGroupByOtherTimeFieldIT extends ContinuousTestCase {
             if (b.getDocCount() == 0) {
                 continue;
             }
-            long second = ((ZonedDateTime) b.getKey()).toEpochSecond() * 1000;
             List<? extends Terms.Bucket> terms = ((Terms) b.getAggregations().get("event")).getBuckets();
             for (Terms.Bucket t : terms) {
-                flattenedBuckets.add(flattenedResult(second, t.getKeyAsString(), t.getDocCount()));
+                flattenedBuckets.add(flattenedResult(b.getKeyAsString(), t.getKeyAsString(), t.getDocCount()));
             }
         }
 
@@ -188,7 +199,13 @@ public class DateHistogramGroupByOtherTimeFieldIT extends ContinuousTestCase {
             SearchHit searchHit = destIterator.next();
             Map<String, Object> source = searchHit.getSourceAsMap();
 
-            Long transformBucketKey = (Long) XContentMapValues.extractValue("second", source);
+            String transformBucketKey;
+            if (datesAsEpochMillis) {
+                transformBucketKey = ContinuousTestCase.STRICT_DATE_OPTIONAL_TIME_PRINTER_NANOS.withZone(ZoneId.of("UTC"))
+                    .format(Instant.ofEpochMilli((Long) XContentMapValues.extractValue("second", source)));
+            } else {
+                transformBucketKey = (String) XContentMapValues.extractValue("second", source);
+            }
 
             // test correctness, the results from the aggregation and the results from the transform should be the same
             assertThat(
@@ -228,7 +245,7 @@ public class DateHistogramGroupByOtherTimeFieldIT extends ContinuousTestCase {
         assertFalse(destIterator.hasNext());
     }
 
-    private static Map<String, Object> flattenedResult(long second, String event, long count) {
+    private static Map<String, Object> flattenedResult(String second, String event, long count) {
         Map<String, Object> doc = new HashMap<>();
         doc.put("second", second);
         doc.put("event", event);

x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TermsOnDateGroupByIT.java

@@ -27,6 +27,8 @@ import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation.S
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 
 import java.io.IOException;
+import java.time.Instant;
+import java.time.ZoneId;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -41,8 +43,8 @@ import static org.hamcrest.Matchers.equalTo;
 public class TermsOnDateGroupByIT extends ContinuousTestCase {
 
     private static final String NAME = "continuous-terms-on-date-pivot-test";
-    private static final long MISSING_BUCKET_KEY = 1262304000000L; // 01/01/2010 should end up last when sorting
-
+    private static final String MISSING_BUCKET_KEY = ContinuousTestCase.STRICT_DATE_OPTIONAL_TIME_PRINTER_NANOS.withZone(ZoneId.of("UTC"))
+        .format(Instant.ofEpochMilli(1262304000000L)); // 01/01/2010 should end up last when sorting
     private final boolean missing;
 
     public TermsOnDateGroupByIT() {
@@ -126,15 +128,16 @@ public class TermsOnDateGroupByIT extends ContinuousTestCase {
             SearchHit searchHit = destIterator.next();
             Map<String, Object> source = searchHit.getSourceAsMap();
 
-            Long transformBucketKey = (Long) XContentMapValues.extractValue("some-timestamp", source);
+            String transformBucketKey = (String) XContentMapValues.extractValue("some-timestamp", source);
+
             if (transformBucketKey == null) {
                 transformBucketKey = MISSING_BUCKET_KEY;
             }
             // test correctness, the results from the aggregation and the results from the transform should be the same
             assertThat(
-                "Buckets did not match, source: " + source + ", expected: " + bucket.getKey() + ", iteration: " + iteration,
+                "Buckets did not match, source: " + source + ", expected: " + bucket.getKeyAsString() + ", iteration: " + iteration,
                 transformBucketKey,
-                equalTo(bucket.getKey())
+                equalTo(bucket.getKeyAsString())
             );
             assertThat(
                 "Doc count did not match, source: " + source + ", expected: " + bucket.getDocCount() + ", iteration: " + iteration,

x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/FunctionFactory.java

@@ -24,7 +24,7 @@ public final class FunctionFactory {
      */
     public static Function create(TransformConfig config) {
         if (config.getPivotConfig() != null) {
-            return new Pivot(config.getPivotConfig(), config.getId());
+            return new Pivot(config.getPivotConfig(), config.getId(), config.getSettings(), config.getVersion());
         } else {
             throw new IllegalArgumentException("unknown transform function");
         }

x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java

@@ -1013,14 +1013,14 @@ public abstract class TransformIndexer extends AsyncTwoPhaseIndexer<TransformInd
     }
 
     private synchronized void startIndexerThreadShutdown() {
-        indexerThreadShuttingDown  = true;
+        indexerThreadShuttingDown = true;
         stopCalledDuringIndexerThreadShutdown = false;
     }
 
     private synchronized void finishIndexerThreadShutdown() {
-        indexerThreadShuttingDown  = false;
+        indexerThreadShuttingDown = false;
         if (stopCalledDuringIndexerThreadShutdown) {
-            doSaveState(IndexerState.STOPPED,  getPosition(), () -> {});
+            doSaveState(IndexerState.STOPPED, getPosition(), () -> {});
         }
     }
 

x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtils.java

@@ -14,6 +14,7 @@ import org.elasticsearch.common.geo.builders.PointBuilder;
 import org.elasticsearch.common.geo.builders.PolygonBuilder;
 import org.elasticsearch.common.geo.parsers.ShapeParser;
 import org.elasticsearch.geometry.Rectangle;
+import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.search.aggregations.Aggregation;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
@@ -64,6 +65,8 @@ public final class AggregationResultUtils {
 
     private static final Map<String, BucketKeyExtractor> BUCKET_KEY_EXTRACTOR_MAP;
     private static final BucketKeyExtractor DEFAULT_BUCKET_KEY_EXTRACTOR = new DefaultBucketKeyExtractor();
+    private static final BucketKeyExtractor DATES_AS_EPOCH_BUCKET_KEY_EXTRACTOR = new DatesAsEpochBucketKeyExtractor();
+
     static {
         Map<String, BucketKeyExtractor> tempMap = new HashMap<>();
         tempMap.put(GeoTileGroupSource.class.getName(), new GeoTileBucketKeyExtractor());
@@ -87,7 +90,8 @@ public final class AggregationResultUtils {
         Collection<AggregationBuilder> aggregationBuilders,
         Collection<PipelineAggregationBuilder> pipelineAggs,
         Map<String, String> fieldTypeMap,
-        TransformIndexerStats stats
+        TransformIndexerStats stats,
+        boolean datesAsEpoch
     ) {
         return agg.getBuckets().stream().map(bucket -> {
             stats.incrementNumDocuments(bucket.getDocCount());
@@ -104,7 +108,7 @@ public final class AggregationResultUtils {
                 updateDocument(
                     document,
                     destinationFieldName,
-                    getBucketKeyExtractor(singleGroupSource).value(value, fieldTypeMap.get(destinationFieldName))
+                    getBucketKeyExtractor(singleGroupSource, datesAsEpoch).value(value, fieldTypeMap.get(destinationFieldName))
                 );
             });
 
@@ -128,8 +132,11 @@ public final class AggregationResultUtils {
         });
     }
 
-    static BucketKeyExtractor getBucketKeyExtractor(SingleGroupSource groupSource) {
-        return BUCKET_KEY_EXTRACTOR_MAP.getOrDefault(groupSource.getClass().getName(), DEFAULT_BUCKET_KEY_EXTRACTOR);
+    static BucketKeyExtractor getBucketKeyExtractor(SingleGroupSource groupSource, boolean datesAsEpoch) {
+        return BUCKET_KEY_EXTRACTOR_MAP.getOrDefault(
+            groupSource.getClass().getName(),
+            datesAsEpoch ? DATES_AS_EPOCH_BUCKET_KEY_EXTRACTOR : DEFAULT_BUCKET_KEY_EXTRACTOR
+        );
     }
 
     static AggValueExtractor getExtractor(Aggregation aggregation) {
@@ -409,6 +416,24 @@ public final class AggregationResultUtils {
 
     static class DefaultBucketKeyExtractor implements BucketKeyExtractor {
 
+        @Override
+        public Object value(Object key, String type) {
+            if (isNumericType(type) && key instanceof Double) {
+                return dropFloatingPointComponentIfTypeRequiresIt(type, (Double) key);
+            } else if ((DateFieldMapper.CONTENT_TYPE.equals(type) || DateFieldMapper.DATE_NANOS_CONTENT_TYPE.equals(type))
+                && key instanceof Long) {
+                    // date_histogram return bucket keys with milliseconds since epoch precision, therefore we don't need a
+                    // nanosecond formatter, for the parser on indexing side, time is optional (only the date part is mandatory)
+                    return DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis((Long) key);
+                }
+
+            return key;
+        }
+
+    }
+
+    static class DatesAsEpochBucketKeyExtractor implements BucketKeyExtractor {
+
         @Override
         public Object value(Object key, String type) {
             if (isNumericType(type) && key instanceof Double) {

x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/Pivot.java

@@ -10,6 +10,7 @@ import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchAction;
@@ -37,6 +38,7 @@ import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.core.transform.TransformField;
 import org.elasticsearch.xpack.core.transform.TransformMessages;
+import org.elasticsearch.xpack.core.transform.transforms.SettingsConfig;
 import org.elasticsearch.xpack.core.transform.transforms.SourceConfig;
 import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats;
 import org.elasticsearch.xpack.core.transform.transforms.TransformProgress;
@@ -63,13 +65,17 @@ public class Pivot implements Function {
 
     private final PivotConfig config;
     private final String transformId;
+    private final SettingsConfig settings;
+    private final Version version;
 
     // objects for re-using
     private final CompositeAggregationBuilder cachedCompositeAggregation;
 
-    public Pivot(PivotConfig config, String transformId) {
+    public Pivot(PivotConfig config, String transformId, SettingsConfig settings, Version version) {
         this.config = config;
         this.transformId = transformId;
+        this.settings = settings;
+        this.version = version == null ? Version.CURRENT : version;
         this.cachedCompositeAggregation = createCompositeAggregation(config);
     }
 
@@ -217,13 +223,22 @@ public class Pivot implements Function {
         Collection<AggregationBuilder> aggregationBuilders = config.getAggregationConfig().getAggregatorFactories();
         Collection<PipelineAggregationBuilder> pipelineAggregationBuilders = config.getAggregationConfig().getPipelineAggregatorFactories();
 
+        // defines how dates are written, if not specified in settings
+        // < 7.11 as epoch millis
+        // >= 7.11 as string
+        // note: it depends on the version when the transform has been created, not the version of the code
+        boolean datesAsEpoch = settings.getDatesAsEpochMillis() != null ? settings.getDatesAsEpochMillis()
+            : version.onOrAfter(Version.V_8_0_0) ? false // todo V_7_11_0
+            : true;
+
         return AggregationResultUtils.extractCompositeAggregationResults(
             agg,
             groups,
             aggregationBuilders,
             pipelineAggregationBuilders,
             fieldTypeMap,
-            transformIndexerStats
+            transformIndexerStats,
+            datesAsEpoch
         );
     }
 

x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java

@@ -465,7 +465,7 @@ public class TransformIndexerStateTests extends ESTestCase {
             null,
             randomPivotConfig(),
             randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000),
-            new SettingsConfig(null, Float.valueOf(1.0f))
+            new SettingsConfig(null, Float.valueOf(1.0f), (Boolean) null)
         );
         AtomicReference<IndexerState> state = new AtomicReference<>(IndexerState.STARTED);
 

x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java

@@ -263,7 +263,7 @@ public class TransformIndexerTests extends ESTestCase {
             null,
             randomPivotConfig(),
             randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000),
-            new SettingsConfig(pageSize, null)
+            new SettingsConfig(pageSize, null, (Boolean) null)
         );
         AtomicReference<IndexerState> state = new AtomicReference<>(IndexerState.STOPPED);
         final long initialPageSize = pageSize == null ? Transform.DEFAULT_INITIAL_MAX_PAGE_SEARCH_SIZE : pageSize;
@@ -331,7 +331,7 @@ public class TransformIndexerTests extends ESTestCase {
             null,
             randomPivotConfig(),
             randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000),
-            new SettingsConfig(pageSize, null)
+            new SettingsConfig(pageSize, null, (Boolean) null)
         );
         SearchResponse searchResponse = new SearchResponse(
             new InternalSearchResponse(
@@ -389,7 +389,7 @@ public class TransformIndexerTests extends ESTestCase {
             null,
             randomPivotConfig(),
             randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000),
-            new SettingsConfig(pageSize, null)
+            new SettingsConfig(pageSize, null, (Boolean) null)
         );
         AtomicReference<IndexerState> state = new AtomicReference<>(IndexerState.STOPPED);
         Function<SearchRequest, SearchResponse> searchFunction = searchRequest -> {

x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtilsTests.java

@@ -62,6 +62,7 @@ import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.transform.TransformField;
 import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats;
 import org.elasticsearch.xpack.core.transform.transforms.pivot.GroupConfig;
+import org.elasticsearch.xpack.transform.transforms.pivot.AggregationResultUtils.BucketKeyExtractor;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -871,6 +872,26 @@ public class AggregationResultUtilsTests extends ESTestCase {
         );
     }
 
+    public void testDefaultBucketKeyExtractor() {
+        BucketKeyExtractor extractor = new AggregationResultUtils.DefaultBucketKeyExtractor();
+
+        assertThat(extractor.value(42.0, "long"), equalTo(42L));
+        assertThat(extractor.value(42.2, "double"), equalTo(42.2));
+        assertThat(extractor.value(1577836800000L, "date"), equalTo("2020-01-01T00:00:00.000Z"));
+        assertThat(extractor.value(1577836800000L, "date_nanos"), equalTo("2020-01-01T00:00:00.000Z"));
+        assertThat(extractor.value(1577836800000L, "long"), equalTo(1577836800000L));
+    }
+
+    public void testDatesAsEpochBucketKeyExtractor() {
+        BucketKeyExtractor extractor = new AggregationResultUtils.DatesAsEpochBucketKeyExtractor();
+
+        assertThat(extractor.value(42.0, "long"), equalTo(42L));
+        assertThat(extractor.value(42.2, "double"), equalTo(42.2));
+        assertThat(extractor.value(1577836800000L, "date"), equalTo(1577836800000L));
+        assertThat(extractor.value(1577836800000L, "date_nanos"), equalTo(1577836800000L));
+        assertThat(extractor.value(1577836800000L, "long"), equalTo(1577836800000L));
+    }
+
     private void executeTest(
         GroupConfig groups,
         Collection<AggregationBuilder> aggregationBuilders,
@@ -923,7 +944,8 @@ public class AggregationResultUtilsTests extends ESTestCase {
                 aggregationBuilders,
                 pipelineAggregationBuilders,
                 fieldTypeMap,
-                stats
+                stats,
+                true
             ).collect(Collectors.toList());
         }
     }

x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/PivotTests.java

@@ -8,6 +8,7 @@ package org.elasticsearch.xpack.transform.transforms.pivot;
 
 import org.apache.lucene.search.TotalHits;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
@@ -29,6 +30,7 @@ import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.client.NoOpClient;
 import org.elasticsearch.xpack.core.transform.transforms.QueryConfig;
+import org.elasticsearch.xpack.core.transform.transforms.SettingsConfig;
 import org.elasticsearch.xpack.core.transform.transforms.SourceConfig;
 import org.elasticsearch.xpack.core.transform.transforms.pivot.AggregationConfig;
 import org.elasticsearch.xpack.core.transform.transforms.pivot.GroupConfigTests;
@@ -93,14 +95,14 @@ public class PivotTests extends ESTestCase {
 
     public void testValidateExistingIndex() throws Exception {
         SourceConfig source = new SourceConfig(new String[] { "existing_source_index" }, QueryConfig.matchAll());
-        Function pivot = new Pivot(getValidPivotConfig(), randomAlphaOfLength(10));
+        Function pivot = new Pivot(getValidPivotConfig(), randomAlphaOfLength(10), new SettingsConfig(), Version.CURRENT);
 
         assertValidTransform(client, source, pivot);
     }
 
     public void testValidateNonExistingIndex() throws Exception {
         SourceConfig source = new SourceConfig(new String[] { "non_existing_source_index" }, QueryConfig.matchAll());
-        Function pivot = new Pivot(getValidPivotConfig(), randomAlphaOfLength(10));
+        Function pivot = new Pivot(getValidPivotConfig(), randomAlphaOfLength(10), new SettingsConfig(), Version.CURRENT);
 
         assertInvalidTransform(client, source, pivot);
     }
@@ -110,13 +112,17 @@ public class PivotTests extends ESTestCase {
 
         Function pivot = new Pivot(
             new PivotConfig(GroupConfigTests.randomGroupConfig(), getValidAggregationConfig(), expectedPageSize),
-            randomAlphaOfLength(10)
+            randomAlphaOfLength(10),
+            new SettingsConfig(),
+            Version.CURRENT
         );
         assertThat(pivot.getInitialPageSize(), equalTo(expectedPageSize));
 
         pivot = new Pivot(
             new PivotConfig(GroupConfigTests.randomGroupConfig(), getValidAggregationConfig(), null),
-            randomAlphaOfLength(10)
+            randomAlphaOfLength(10),
+            new SettingsConfig(),
+            Version.CURRENT
         );
         assertThat(pivot.getInitialPageSize(), equalTo(Transform.DEFAULT_INITIAL_MAX_PAGE_SEARCH_SIZE));
 
@@ -128,7 +134,7 @@ public class PivotTests extends ESTestCase {
         // search has failures although they might just be temporary
         SourceConfig source = new SourceConfig(new String[] { "existing_source_index_with_failing_shards" }, QueryConfig.matchAll());
 
-        Function pivot = new Pivot(getValidPivotConfig(), randomAlphaOfLength(10));
+        Function pivot = new Pivot(getValidPivotConfig(), randomAlphaOfLength(10), new SettingsConfig(), Version.CURRENT);
 
         assertInvalidTransform(client, source, pivot);
     }
@@ -138,7 +144,12 @@ public class PivotTests extends ESTestCase {
             AggregationConfig aggregationConfig = getAggregationConfig(agg);
             SourceConfig source = new SourceConfig(new String[] { "existing_source" }, QueryConfig.matchAll());
 
-            Function pivot = new Pivot(getValidPivotConfig(aggregationConfig), randomAlphaOfLength(10));
+            Function pivot = new Pivot(
+                getValidPivotConfig(aggregationConfig),
+                randomAlphaOfLength(10),
+                new SettingsConfig(),
+                Version.CURRENT
+            );
             assertValidTransform(client, source, pivot);
         }
     }
@@ -147,7 +158,12 @@ public class PivotTests extends ESTestCase {
         for (String agg : unsupportedAggregations) {
             AggregationConfig aggregationConfig = getAggregationConfig(agg);
 
-            Function pivot = new Pivot(getValidPivotConfig(aggregationConfig), randomAlphaOfLength(10));
+            Function pivot = new Pivot(
+                getValidPivotConfig(aggregationConfig),
+                randomAlphaOfLength(10),
+                new SettingsConfig(),
+                Version.CURRENT
+            );
 
             pivot.validateConfig(ActionListener.wrap(r -> { fail("expected an exception but got a response"); }, e -> {
                 assertThat(e, anyOf(instanceOf(ElasticsearchException.class)));