[ML] cleanup + adding description field to transforms (#41554)

* [ML] cleanup + adding description field to transforms

* making description length have a max of 1k

client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfig.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.client.dataframe.transforms;
 
 import org.elasticsearch.client.dataframe.transforms.pivot.PivotConfig;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
@@ -38,6 +39,7 @@ public class DataFrameTransformConfig implements ToXContentObject {
     public static final ParseField ID = new ParseField("id");
     public static final ParseField SOURCE = new ParseField("source");
     public static final ParseField DEST = new ParseField("dest");
+    public static final ParseField DESCRIPTION = new ParseField("description");
     // types of transforms
     public static final ParseField PIVOT_TRANSFORM = new ParseField("pivot");
 
@@ -45,6 +47,7 @@ public class DataFrameTransformConfig implements ToXContentObject {
     private final SourceConfig source;
     private final DestConfig dest;
     private final PivotConfig pivotConfig;
+    private final String description;
 
     public static final ConstructingObjectParser<DataFrameTransformConfig, Void> PARSER =
             new ConstructingObjectParser<>("data_frame_transform", true,
@@ -53,7 +56,8 @@ public class DataFrameTransformConfig implements ToXContentObject {
                     SourceConfig source = (SourceConfig) args[1];
                     DestConfig dest = (DestConfig) args[2];
                     PivotConfig pivotConfig = (PivotConfig) args[3];
-                    return new DataFrameTransformConfig(id, source, dest, pivotConfig);
+                    String description = (String)args[4];
+                    return new DataFrameTransformConfig(id, source, dest, pivotConfig, description);
                 });
 
     static {
@@ -61,21 +65,38 @@ public class DataFrameTransformConfig implements ToXContentObject {
         PARSER.declareObject(constructorArg(), (p, c) -> SourceConfig.PARSER.apply(p, null), SOURCE);
         PARSER.declareObject(constructorArg(), (p, c) -> DestConfig.PARSER.apply(p, null), DEST);
         PARSER.declareObject(optionalConstructorArg(), (p, c) -> PivotConfig.fromXContent(p), PIVOT_TRANSFORM);
+        PARSER.declareString(optionalConstructorArg(), DESCRIPTION);
     }
 
     public static DataFrameTransformConfig fromXContent(final XContentParser parser) {
         return PARSER.apply(parser, null);
     }
 
+    /**
+     * Helper method for previewing a data frame transform configuration
+     *
+     * The DataFrameTransformConfig returned from this method should only be used for previewing the resulting data.
+     *
+     * A new, valid, DataFrameTransformConfig with an appropriate destination and ID will have to be constructed to create
+     * the transform.
+     * @param source Source configuration for gathering the data
+     * @param pivotConfig Pivot config to preview
+     * @return A DataFrameTransformConfig to preview, NOTE it will have a {@code null} id, destination and index.
+     */
+    public static DataFrameTransformConfig forPreview(final SourceConfig source, final PivotConfig pivotConfig) {
+        return new DataFrameTransformConfig(null, source, null, pivotConfig, null);
+    }
 
     public DataFrameTransformConfig(final String id,
                                     final SourceConfig source,
                                     final DestConfig dest,
-                                    final PivotConfig pivotConfig) {
+                                    final PivotConfig pivotConfig,
+                                    final String description) {
         this.id = id;
         this.source = source;
         this.dest = dest;
         this.pivotConfig = pivotConfig;
+        this.description = description;
     }
 
     public String getId() {
@@ -94,6 +115,11 @@ public class DataFrameTransformConfig implements ToXContentObject {
         return pivotConfig;
     }
 
+    @Nullable
+    public String getDescription() {
+        return description;
+    }
+
     @Override
     public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
         builder.startObject();
@@ -109,6 +135,9 @@ public class DataFrameTransformConfig implements ToXContentObject {
         if (pivotConfig != null) {
             builder.field(PIVOT_TRANSFORM.getPreferredName(), pivotConfig);
         }
+        if (description != null) {
+            builder.field(DESCRIPTION.getPreferredName(), description);
+        }
         builder.endObject();
         return builder;
     }
@@ -128,12 +157,13 @@ public class DataFrameTransformConfig implements ToXContentObject {
         return Objects.equals(this.id, that.id)
                 && Objects.equals(this.source, that.source)
                 && Objects.equals(this.dest, that.dest)
+                && Objects.equals(this.description, that.description)
                 && Objects.equals(this.pivotConfig, that.pivotConfig);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(id, source, dest, pivotConfig);
+        return Objects.hash(id, source, dest, pivotConfig, description);
     }
 
     @Override

client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java

@@ -312,7 +312,8 @@ public class DataFrameTransformIT extends ESRestHighLevelClientTestCase {
         return new DataFrameTransformConfig(id,
                 new SourceConfig(new String[]{source}, queryConfig),
                 destConfig,
-                pivotConfig);
+                pivotConfig,
+            "this is a test transform");
     }
 
     public void testGetStats() throws Exception {
@@ -329,7 +330,10 @@ public class DataFrameTransformIT extends ESRestHighLevelClientTestCase {
 
         String id = "test-get-stats";
         DataFrameTransformConfig transform = new DataFrameTransformConfig(id,
-                new SourceConfig(new String[]{sourceIndex}, queryConfig), new DestConfig("pivot-dest"), pivotConfig);
+            new SourceConfig(new String[]{sourceIndex}, queryConfig),
+            new DestConfig("pivot-dest"),
+            pivotConfig,
+            "transform for testing stats");
 
         DataFrameClient client = highLevelClient().dataFrame();
         AcknowledgedResponse ack = execute(new PutDataFrameTransformRequest(transform), client::putDataFrameTransform,

client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/PreviewDataFrameTransformRequestTests.java

@@ -65,13 +65,12 @@ public class PreviewDataFrameTransformRequestTests extends AbstractXContentTestC
                 containsString("preview requires a non-null data frame config"));
 
         // null id and destination is valid
-        DataFrameTransformConfig config = new DataFrameTransformConfig(null, randomSourceConfig(), null,
-            PivotConfigTests.randomPivotConfig());
+        DataFrameTransformConfig config = DataFrameTransformConfig.forPreview(randomSourceConfig(), PivotConfigTests.randomPivotConfig());
 
         assertFalse(new PreviewDataFrameTransformRequest(config).validate().isPresent());
 
         // null source is not valid
-        config = new DataFrameTransformConfig(null, null, null, PivotConfigTests.randomPivotConfig());
+        config = new DataFrameTransformConfig(null, null, null, PivotConfigTests.randomPivotConfig(), null);
 
         Optional<ValidationException> error = new PreviewDataFrameTransformRequest(config).validate();
         assertTrue(error.isPresent());

client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/PutDataFrameTransformRequestTests.java

@@ -40,7 +40,7 @@ public class PutDataFrameTransformRequestTests extends AbstractXContentTestCase<
     public void testValidate() {
         assertFalse(createTestInstance().validate().isPresent());
 
-        DataFrameTransformConfig config = new DataFrameTransformConfig(null, null, null, PivotConfigTests.randomPivotConfig());
+        DataFrameTransformConfig config = new DataFrameTransformConfig(null, null, null, PivotConfigTests.randomPivotConfig(), null);
 
         Optional<ValidationException> error = new PutDataFrameTransformRequest(config).validate();
         assertTrue(error.isPresent());

client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfigTests.java

@@ -37,7 +37,7 @@ public class DataFrameTransformConfigTests extends AbstractXContentTestCase<Data
 
     public static DataFrameTransformConfig randomDataFrameTransformConfig() {
         return new DataFrameTransformConfig(randomAlphaOfLengthBetween(1, 10), randomSourceConfig(),
-                randomDestConfig(), PivotConfigTests.randomPivotConfig());
+                randomDestConfig(), PivotConfigTests.randomPivotConfig(), randomBoolean() ? null : randomAlphaOfLengthBetween(1, 100));
     }
 
     @Override

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java

@@ -141,7 +141,8 @@ public class DataFrameTransformDocumentationIT extends ESRestHighLevelClientTest
                 new DataFrameTransformConfig("reviewer-avg-rating", // <1>
                 sourceConfig, // <2>
                 new DestConfig("pivot-destination"),  // <3>
-                pivotConfig);  // <4>
+                pivotConfig, // <4>
+                "This is my test transform");  // <5>
         // end::put-data-frame-transform-config
 
         {
@@ -161,7 +162,7 @@ public class DataFrameTransformDocumentationIT extends ESRestHighLevelClientTest
         {
             DataFrameTransformConfig configWithDifferentId = new DataFrameTransformConfig("reviewer-avg-rating2",
                     transformConfig.getSource(), transformConfig.getDestination(),
-                    transformConfig.getPivotConfig());
+                    transformConfig.getPivotConfig(), null);
             PutDataFrameTransformRequest request = new PutDataFrameTransformRequest(configWithDifferentId);
 
             // tag::put-data-frame-transform-execute-listener
@@ -205,7 +206,7 @@ public class DataFrameTransformDocumentationIT extends ESRestHighLevelClientTest
         PivotConfig pivotConfig = new PivotConfig(groupConfig, aggConfig);
 
         DataFrameTransformConfig transformConfig = new DataFrameTransformConfig("mega-transform",
-                new SourceConfig(new String[]{"source-data"}, queryConfig), new DestConfig("pivot-dest"), pivotConfig);
+                new SourceConfig(new String[]{"source-data"}, queryConfig), new DestConfig("pivot-dest"), pivotConfig, null);
 
         client.dataFrame().putDataFrameTransform(new PutDataFrameTransformRequest(transformConfig), RequestOptions.DEFAULT);
         transformsToClean.add(transformConfig.getId());
@@ -320,9 +321,9 @@ public class DataFrameTransformDocumentationIT extends ESRestHighLevelClientTest
         PivotConfig pivotConfig = new PivotConfig(groupConfig, aggConfig);
 
         DataFrameTransformConfig transformConfig1 = new DataFrameTransformConfig("mega-transform",
-                new SourceConfig(new String[]{"source-data"}, queryConfig), new DestConfig("pivot-dest"), pivotConfig);
+                new SourceConfig(new String[]{"source-data"}, queryConfig), new DestConfig("pivot-dest"), pivotConfig, null);
         DataFrameTransformConfig transformConfig2 = new DataFrameTransformConfig("mega-transform2",
-                new SourceConfig(new String[]{"source-data"}, queryConfig), new DestConfig("pivot-dest2"), pivotConfig);
+                new SourceConfig(new String[]{"source-data"}, queryConfig), new DestConfig("pivot-dest2"), pivotConfig, null);
 
         client.dataFrame().putDataFrameTransform(new PutDataFrameTransformRequest(transformConfig1), RequestOptions.DEFAULT);
         client.dataFrame().putDataFrameTransform(new PutDataFrameTransformRequest(transformConfig2), RequestOptions.DEFAULT);
@@ -386,11 +387,9 @@ public class DataFrameTransformDocumentationIT extends ESRestHighLevelClientTest
 
         // tag::preview-data-frame-transform-request
         DataFrameTransformConfig transformConfig =
-                new DataFrameTransformConfig(null,  // <1>
-                new SourceConfig(new String[]{"source-data"}, queryConfig),
-                null,                               // <2>
-                pivotConfig);
-
+            DataFrameTransformConfig.forPreview(
+                new SourceConfig(new String[]{"source-data"}, queryConfig), // <1>
+                pivotConfig); // <2>
         PreviewDataFrameTransformRequest request =
                 new PreviewDataFrameTransformRequest(transformConfig); // <3>
         // end::preview-data-frame-transform-request
@@ -447,7 +446,7 @@ public class DataFrameTransformDocumentationIT extends ESRestHighLevelClientTest
 
         String id = "statisitcal-transform";
         DataFrameTransformConfig transformConfig = new DataFrameTransformConfig(id,
-                new SourceConfig(new String[]{"source-data"}, queryConfig), new DestConfig("dest"), pivotConfig);
+                new SourceConfig(new String[]{"source-data"}, queryConfig), new DestConfig("dest"), pivotConfig, null);
         client.dataFrame().putDataFrameTransform(new PutDataFrameTransformRequest(transformConfig), RequestOptions.DEFAULT);
 
         // tag::get-data-frame-transform-stats-request
@@ -526,7 +525,7 @@ public class DataFrameTransformDocumentationIT extends ESRestHighLevelClientTest
         
         DataFrameTransformConfig putTransformConfig = new DataFrameTransformConfig("mega-transform",
                 new SourceConfig(new String[]{"source-data"}, queryConfig),
-                new DestConfig("pivot-dest"), pivotConfig);
+                new DestConfig("pivot-dest"), pivotConfig, null);
 
         RestHighLevelClient client = highLevelClient();
         client.dataFrame().putDataFrameTransform(new PutDataFrameTransformRequest(putTransformConfig), RequestOptions.DEFAULT);

docs/java-rest/high-level/dataframe/preview_data_frame.asciidoc

@@ -20,8 +20,8 @@ A +{request}+ takes a single argument: a valid data frame transform config.
 --------------------------------------------------
 include-tagged::{doc-tests-file}[{api}-request]
 --------------------------------------------------
-<1> The transform Id may be null for the preview
-<2> The destination may be null for the preview
+<1> The source config from which the data should be gathered
+<2> The pivot config used to transform the data
 <3> The configuration of the {dataframe-job} to preview
 
 include::../execution.asciidoc[]

docs/java-rest/high-level/dataframe/put_data_frame.asciidoc

@@ -35,6 +35,7 @@ include-tagged::{doc-tests-file}[{api}-config]
 <2> The source indices and query from which to gather data
 <3> The destination index
 <4> The PivotConfig
+<5> Optional free text description of the transform
 
 [id="{upid}-{api}-query-config"]
 

docs/reference/data-frames/apis/put-transform.asciidoc

@@ -33,6 +33,8 @@ a `query`.
 `pivot`:: Defines the pivot function `group by` fields and the aggregation to
 reduce the data.
 
+`description`:: Optional free text description of the data frame transform
+
 
 //==== Authorization
 
@@ -73,7 +75,8 @@ PUT _data_frame/transforms/ecommerce_transform
         }
       }
     }
-  }
+  },
+  "description": "Maximum priced ecommerce data by customer_id in Asia"
 }
 --------------------------------------------------
 // CONSOLE

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/DeleteDataFrameTransformAction.java

@@ -18,7 +18,7 @@ import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.xpack.core.dataframe.DataFrameField;
-import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.core.dataframe.utils.ExceptionsHelper;
 
 import java.io.IOException;
 import java.util.Collections;

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/StartDataFrameTransformAction.java

@@ -16,7 +16,7 @@ import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.core.dataframe.DataFrameField;
-import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.core.dataframe.utils.ExceptionsHelper;
 
 import java.io.IOException;
 import java.util.Collections;

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/StartDataFrameTransformTaskAction.java

@@ -17,7 +17,7 @@ import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.xpack.core.dataframe.DataFrameField;
-import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.core.dataframe.utils.ExceptionsHelper;
 
 import java.io.IOException;
 import java.util.Collections;

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/StopDataFrameTransformAction.java

@@ -18,7 +18,7 @@ import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.xpack.core.dataframe.DataFrameField;
-import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.core.dataframe.utils.ExceptionsHelper;
 
 import java.io.IOException;
 import java.util.Arrays;

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/notifications/DataFrameAuditMessage.java

@@ -12,7 +12,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.xpack.core.common.notifications.AbstractAuditMessage;
 import org.elasticsearch.xpack.core.common.notifications.Level;
 import org.elasticsearch.xpack.core.dataframe.DataFrameField;
-import org.elasticsearch.xpack.core.ml.utils.time.TimeUtils;
+import org.elasticsearch.xpack.core.dataframe.utils.TimeUtils;
 
 import java.util.Date;
 
@@ -36,15 +36,10 @@ public class DataFrameAuditMessage extends AbstractAuditMessage {
             }
             throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
         }, LEVEL, ObjectParser.ValueType.STRING);
-        PARSER.declareField(constructorArg(), parser -> {
-            if (parser.currentToken() == XContentParser.Token.VALUE_NUMBER) {
-                return new Date(parser.longValue());
-            } else if (parser.currentToken() == XContentParser.Token.VALUE_STRING) {
-                return new Date(TimeUtils.dateStringToEpoch(parser.text()));
-            }
-            throw new IllegalArgumentException(
-                "unexpected token [" + parser.currentToken() + "] for [" + TIMESTAMP.getPreferredName() + "]");
-        }, TIMESTAMP, ObjectParser.ValueType.VALUE);
+        PARSER.declareField(constructorArg(),
+            p -> TimeUtils.parseTimeField(p, TIMESTAMP.getPreferredName()),
+            TIMESTAMP,
+            ObjectParser.ValueType.VALUE);
         PARSER.declareString(optionalConstructorArg(), NODE_NAME);
     }
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfig.java

@@ -20,7 +20,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.xpack.core.dataframe.DataFrameField;
 import org.elasticsearch.xpack.core.dataframe.DataFrameMessages;
 import org.elasticsearch.xpack.core.dataframe.transforms.pivot.PivotConfig;
-import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.core.dataframe.utils.ExceptionsHelper;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -41,13 +41,15 @@ public class DataFrameTransformConfig extends AbstractDiffable<DataFrameTransfor
     // types of transforms
     public static final ParseField PIVOT_TRANSFORM = new ParseField("pivot");
 
+    public static final ParseField DESCRIPTION = new ParseField("description");
     private static final ConstructingObjectParser<DataFrameTransformConfig, String> STRICT_PARSER = createParser(false);
     private static final ConstructingObjectParser<DataFrameTransformConfig, String> LENIENT_PARSER = createParser(true);
+    private static final int MAX_DESCRIPTION_LENGTH = 1_000;
 
     private final String id;
     private final SourceConfig source;
     private final DestConfig dest;
-
+    private final String description;
     // headers store the user context from the creating user, which allows us to run the transform as this user
     // the header only contains name, groups and other context but no authorization keys
     private Map<String, String> headers;
@@ -81,7 +83,8 @@ public class DataFrameTransformConfig extends AbstractDiffable<DataFrameTransfor
                     Map<String, String> headers = (Map<String, String>) args[4];
 
                     PivotConfig pivotConfig = (PivotConfig) args[5];
-                    return new DataFrameTransformConfig(id, source, dest, headers, pivotConfig);
+                    String description = (String)args[6];
+                    return new DataFrameTransformConfig(id, source, dest, headers, pivotConfig, description);
                 });
 
         parser.declareString(optionalConstructorArg(), DataFrameField.ID);
@@ -91,6 +94,7 @@ public class DataFrameTransformConfig extends AbstractDiffable<DataFrameTransfor
         parser.declareString(optionalConstructorArg(), DataFrameField.INDEX_DOC_TYPE);
         parser.declareObject(optionalConstructorArg(), (p, c) -> p.mapStrings(), HEADERS);
         parser.declareObject(optionalConstructorArg(), (p, c) -> PivotConfig.fromXContent(p, lenient), PIVOT_TRANSFORM);
+        parser.declareString(optionalConstructorArg(), DESCRIPTION);
 
         return parser;
     }
@@ -103,17 +107,22 @@ public class DataFrameTransformConfig extends AbstractDiffable<DataFrameTransfor
                                     final SourceConfig source,
                                     final DestConfig dest,
                                     final Map<String, String> headers,
-                                    final PivotConfig pivotConfig) {
+                                    final PivotConfig pivotConfig,
+                                    final String description) {
         this.id = ExceptionsHelper.requireNonNull(id, DataFrameField.ID.getPreferredName());
         this.source = ExceptionsHelper.requireNonNull(source, DataFrameField.SOURCE.getPreferredName());
         this.dest = ExceptionsHelper.requireNonNull(dest, DataFrameField.DESTINATION.getPreferredName());
         this.setHeaders(headers == null ? Collections.emptyMap() : headers);
         this.pivotConfig = pivotConfig;
+        this.description = description;
 
         // at least one function must be defined
         if (this.pivotConfig == null) {
             throw new IllegalArgumentException(DataFrameMessages.DATA_FRAME_TRANSFORM_CONFIGURATION_NO_TRANSFORM);
         }
+        if (this.description != null && this.description.length() > MAX_DESCRIPTION_LENGTH) {
+            throw new IllegalArgumentException("[description] must be less than 1000 characters in length.");
+        }
     }
 
     public DataFrameTransformConfig(final StreamInput in) throws IOException {
@@ -122,6 +131,7 @@ public class DataFrameTransformConfig extends AbstractDiffable<DataFrameTransfor
         dest = new DestConfig(in);
         setHeaders(in.readMap(StreamInput::readString, StreamInput::readString));
         pivotConfig = in.readOptionalWriteable(PivotConfig::new);
+        description = in.readOptionalString();
     }
 
     public String getId() {
@@ -148,6 +158,11 @@ public class DataFrameTransformConfig extends AbstractDiffable<DataFrameTransfor
         return pivotConfig;
     }
 
+    @Nullable
+    public String getDescription() {
+        return description;
+    }
+
     public boolean isValid() {
         if (pivotConfig != null && pivotConfig.isValid() == false) {
             return false;
@@ -163,6 +178,7 @@ public class DataFrameTransformConfig extends AbstractDiffable<DataFrameTransfor
         dest.writeTo(out);
         out.writeMap(headers, StreamOutput::writeString, StreamOutput::writeString);
         out.writeOptionalWriteable(pivotConfig);
+        out.writeOptionalString(description);
     }
 
     @Override
@@ -180,7 +196,9 @@ public class DataFrameTransformConfig extends AbstractDiffable<DataFrameTransfor
         if (headers.isEmpty() == false && params.paramAsBoolean(DataFrameField.FOR_INTERNAL_STORAGE, false) == true) {
             builder.field(HEADERS.getPreferredName(), headers);
         }
-
+        if (description != null) {
+            builder.field(DESCRIPTION.getPreferredName(), description);
+        }
         builder.endObject();
         return builder;
     }
@@ -201,12 +219,13 @@ public class DataFrameTransformConfig extends AbstractDiffable<DataFrameTransfor
                 && Objects.equals(this.source, that.source)
                 && Objects.equals(this.dest, that.dest)
                 && Objects.equals(this.headers, that.headers)
-                && Objects.equals(this.pivotConfig, that.pivotConfig);
+                && Objects.equals(this.pivotConfig, that.pivotConfig)
+                && Objects.equals(this.description, that.description);
     }
 
     @Override
     public int hashCode(){
-        return Objects.hash(id, source, dest, headers, pivotConfig);
+        return Objects.hash(id, source, dest, headers, pivotConfig, description);
     }
 
     @Override

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DestConfig.java

@@ -14,7 +14,7 @@ import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.core.dataframe.utils.ExceptionsHelper;
 
 import java.io.IOException;
 import java.util.Objects;

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/SourceConfig.java

@@ -15,7 +15,7 @@ import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.core.dataframe.utils.ExceptionsHelper;
 
 import java.io.IOException;
 import java.util.Arrays;

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/pivot/GroupConfig.java

@@ -22,7 +22,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.xpack.core.dataframe.DataFrameField;
 import org.elasticsearch.xpack.core.dataframe.DataFrameMessages;
-import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.core.dataframe.utils.ExceptionsHelper;
 
 import java.io.IOException;
 import java.util.LinkedHashMap;

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/pivot/PivotConfig.java

@@ -15,7 +15,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregationBuilder;
 import org.elasticsearch.xpack.core.dataframe.DataFrameField;
-import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.core.dataframe.utils.ExceptionsHelper;
 
 import java.io.IOException;
 import java.util.Map.Entry;

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/utils/TimeUtils.java

@@ -0,0 +1,129 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.dataframe.utils;
+
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.mapper.DateFieldMapper;
+
+import java.io.IOException;
+import java.util.Date;
+import java.util.concurrent.TimeUnit;
+
+public final class TimeUtils {
+
+    private TimeUtils() {
+        // Do nothing
+    }
+
+    public static Date parseTimeField(XContentParser parser, String fieldName) throws IOException {
+        if (parser.currentToken() == XContentParser.Token.VALUE_NUMBER) {
+            return new Date(parser.longValue());
+        } else if (parser.currentToken() == XContentParser.Token.VALUE_STRING) {
+            return new Date(TimeUtils.dateStringToEpoch(parser.text()));
+        }
+        throw new IllegalArgumentException(
+                "unexpected token [" + parser.currentToken() + "] for [" + fieldName + "]");
+    }
+
+    /**
+     * First tries to parse the date first as a Long and convert that to an
+     * epoch time. If the long number has more than 10 digits it is considered a
+     * time in milliseconds else if 10 or less digits it is in seconds. If that
+     * fails it tries to parse the string using
+     * {@link DateFieldMapper#DEFAULT_DATE_TIME_FORMATTER}
+     *
+     * If the date string cannot be parsed -1 is returned.
+     *
+     * @return The epoch time in milliseconds or -1 if the date cannot be
+     *         parsed.
+     */
+    public static long dateStringToEpoch(String date) {
+        try {
+            long epoch = Long.parseLong(date);
+            if (date.trim().length() <= 10) { // seconds
+                return epoch * 1000;
+            } else {
+                return epoch;
+            }
+        } catch (NumberFormatException nfe) {
+            // not a number
+        }
+
+        try {
+            return DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis(date);
+        } catch (ElasticsearchParseException | IllegalArgumentException e) {
+        }
+        // Could not do the conversion
+        return -1;
+    }
+
+    /**
+     * Checks that the given {@code timeValue} is a non-negative multiple value of the {@code baseUnit}.
+     *
+     * <ul>
+     *   <li>400ms is valid for base unit of seconds</li>
+     *   <li>450ms is invalid for base unit of seconds but valid for base unit of milliseconds</li>
+     * </ul>
+     */
+    public static void checkNonNegativeMultiple(TimeValue timeValue, TimeUnit baseUnit, ParseField field) {
+        checkNonNegative(timeValue, field);
+        checkMultiple(timeValue, baseUnit, field);
+    }
+
+    /**
+     * Checks that the given {@code timeValue} is a positive multiple value of the {@code baseUnit}.
+     *
+     * <ul>
+     *   <li>400ms is valid for base unit of seconds</li>
+     *   <li>450ms is invalid for base unit of seconds but valid for base unit of milliseconds</li>
+     * </ul>
+     */
+    public static void checkPositiveMultiple(TimeValue timeValue, TimeUnit baseUnit, ParseField field) {
+        checkPositive(timeValue, field);
+        checkMultiple(timeValue, baseUnit, field);
+    }
+
+    /**
+     * Checks that the given {@code timeValue} is positive.
+     *
+     * <ul>
+     *   <li>1s is valid</li>
+     *   <li>-1s is invalid</li>
+     * </ul>
+     */
+    public static void checkPositive(TimeValue timeValue, ParseField field) {
+        long nanos = timeValue.getNanos();
+        if (nanos <= 0) {
+            throw new IllegalArgumentException(field.getPreferredName() + " cannot be less or equal than 0. Value = "
+                    + timeValue.toString());
+        }
+    }
+
+    private static void checkNonNegative(TimeValue timeValue, ParseField field) {
+        long nanos = timeValue.getNanos();
+        if (nanos < 0) {
+            throw new IllegalArgumentException(field.getPreferredName() + " cannot be less than 0. Value = " + timeValue.toString());
+        }
+    }
+
+
+
+    /**
+     * Check the given {@code timeValue} is a multiple of the {@code baseUnit}
+     */
+    public static void checkMultiple(TimeValue timeValue, TimeUnit baseUnit, ParseField field) {
+        long nanos = timeValue.getNanos();
+        TimeValue base = new TimeValue(1, baseUnit);
+        long baseNanos = base.getNanos();
+        if (nanos % baseNanos != 0) {
+            throw new IllegalArgumentException(field.getPreferredName() + " has to be a multiple of " + base.toString() + "; actual was '"
+                    + timeValue.toString() + "'");
+        }
+    }
+}

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/PreviewDataFrameTransformActionRequestTests.java

@@ -68,7 +68,7 @@ public class PreviewDataFrameTransformActionRequestTests extends AbstractSeriali
     @Override
     protected Request createTestInstance() {
         DataFrameTransformConfig config = new DataFrameTransformConfig("transform-preview", randomSourceConfig(),
-                new DestConfig("unused-transform-preview-index"), null, PivotConfigTests.randomPivotConfig());
+                new DestConfig("unused-transform-preview-index"), null, PivotConfigTests.randomPivotConfig(), null);
         return new Request(config);
     }
 

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfigTests.java

@@ -25,6 +25,7 @@ import static org.elasticsearch.test.TestMatchers.matchesPattern;
 import static org.elasticsearch.xpack.core.dataframe.transforms.DestConfigTests.randomDestConfig;
 import static org.elasticsearch.xpack.core.dataframe.transforms.SourceConfigTests.randomInvalidSourceConfig;
 import static org.elasticsearch.xpack.core.dataframe.transforms.SourceConfigTests.randomSourceConfig;
+import static org.hamcrest.Matchers.equalTo;
 
 public class DataFrameTransformConfigTests extends AbstractSerializingDataFrameTestCase<DataFrameTransformConfig> {
 
@@ -41,21 +42,23 @@ public class DataFrameTransformConfigTests extends AbstractSerializingDataFrameT
 
     public static DataFrameTransformConfig randomDataFrameTransformConfigWithoutHeaders(String id) {
         return new DataFrameTransformConfig(id, randomSourceConfig(), randomDestConfig(), null,
-                PivotConfigTests.randomPivotConfig());
+                PivotConfigTests.randomPivotConfig(), randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000));
     }
 
     public static DataFrameTransformConfig randomDataFrameTransformConfig(String id) {
         return new DataFrameTransformConfig(id, randomSourceConfig(), randomDestConfig(), randomHeaders(),
-                PivotConfigTests.randomPivotConfig());
+                PivotConfigTests.randomPivotConfig(), randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000));
     }
 
     public static DataFrameTransformConfig randomInvalidDataFrameTransformConfig() {
         if (randomBoolean()) {
             return new DataFrameTransformConfig(randomAlphaOfLengthBetween(1, 10), randomInvalidSourceConfig(),
-                    randomDestConfig(), randomHeaders(), PivotConfigTests.randomPivotConfig());
+                    randomDestConfig(), randomHeaders(), PivotConfigTests.randomPivotConfig(),
+                randomBoolean() ? null : randomAlphaOfLengthBetween(1, 100));
         } // else
         return new DataFrameTransformConfig(randomAlphaOfLengthBetween(1, 10), randomSourceConfig(),
-                randomDestConfig(), randomHeaders(), PivotConfigTests.randomInvalidPivotConfig());
+                randomDestConfig(), randomHeaders(), PivotConfigTests.randomInvalidPivotConfig(),
+            randomBoolean() ? null : randomAlphaOfLengthBetween(1, 100));
     }
 
     @Before
@@ -162,6 +165,16 @@ public class DataFrameTransformConfigTests extends AbstractSerializingDataFrameT
         }
     }
 
+    public void testMaxLengthDescription() {
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> new DataFrameTransformConfig("id",
+            randomSourceConfig(), randomDestConfig(), null, PivotConfigTests.randomPivotConfig(), randomAlphaOfLength(1001)));
+        assertThat(exception.getMessage(), equalTo("[description] must be less than 1000 characters in length."));
+        String description = randomAlphaOfLength(1000);
+        DataFrameTransformConfig config = new DataFrameTransformConfig("id",
+            randomSourceConfig(), randomDestConfig(), null, PivotConfigTests.randomPivotConfig(), description);
+        assertThat(description, equalTo(config.getDescription()));
+    }
+
     public void testSetIdInBody() throws IOException {
         String pivotTransform = "{"
                 + " \"id\" : \"body_id\","
@@ -189,6 +202,7 @@ public class DataFrameTransformConfigTests extends AbstractSerializingDataFrameT
                 ex.getCause().getMessage());
     }
 
+
     private DataFrameTransformConfig createDataFrameTransformConfigFromString(String json, String id) throws IOException {
         final XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry(),
                 DeprecationHandler.THROW_UNSUPPORTED_OPERATION, json);

x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameIntegTestCase.java

@@ -193,7 +193,8 @@ abstract class DataFrameIntegTestCase extends ESIntegTestCase {
             new SourceConfig(sourceIndices, createQueryConfig(queryBuilder)),
             new DestConfig(destinationIndex),
             Collections.emptyMap(),
-            createPivotConfig(groups, aggregations));
+            createPivotConfig(groups, aggregations),
+            "Test data frame transform config id: " + id);
     }
 
     protected void createReviewsIndex() throws Exception {

x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameGetAndGetStatsIT.java

@@ -57,6 +57,7 @@ public class DataFrameGetAndGetStatsIT extends DataFrameRestTestCase {
         wipeDataFrameTransforms();
     }
 
+    @SuppressWarnings("unchecked")
     public void testGetAndGetStats() throws Exception {
         createPivotReviewsTransform("pivot_1", "pivot_reviews_1", null);
         createPivotReviewsTransform("pivot_2", "pivot_reviews_2", null);

x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformProgressIT.java

@@ -135,7 +135,8 @@ public class DataFrameTransformProgressIT extends ESIntegTestCase {
             sourceConfig,
             destConfig,
             null,
-            pivotConfig);
+            pivotConfig,
+            null);
 
         PlainActionFuture<DataFrameTransformProgress> progressFuture = new PlainActionFuture<>();
         TransformProgressGatherer.getInitialProgress(client(), config, progressFuture);
@@ -154,7 +155,8 @@ public class DataFrameTransformProgressIT extends ESIntegTestCase {
             sourceConfig,
             destConfig,
             null,
-            pivotConfig);
+            pivotConfig,
+            null);
 
 
         progressFuture = new PlainActionFuture<>();

x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataFrameInternalIndex.java

@@ -16,6 +16,7 @@ import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.xpack.core.common.notifications.AbstractAuditMessage;
 import org.elasticsearch.xpack.core.dataframe.DataFrameField;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameIndexerTransformStats;
+import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformConfig;
 import org.elasticsearch.xpack.core.dataframe.transforms.DestConfig;
 import org.elasticsearch.xpack.core.dataframe.transforms.SourceConfig;
 
@@ -194,6 +195,9 @@ public final class DataFrameInternalIndex {
                         .field(TYPE, KEYWORD)
                     .endObject()
                 .endObject()
+            .endObject()
+            .startObject(DataFrameTransformConfig.DESCRIPTION.getPreferredName())
+                .field(TYPE, TEXT)
             .endObject();
     }
 

x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/action/TransportStopDataFrameTransformActionTests.java

@@ -6,16 +6,19 @@
 
 package org.elasticsearch.xpack.dataframe.action;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.persistent.PersistentTaskParams;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.dataframe.DataFrameField;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransform;
-import org.elasticsearch.xpack.core.ml.MlTasks;
-import org.elasticsearch.xpack.core.ml.action.OpenJobAction;
 
+import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collections;
 
@@ -34,8 +37,28 @@ public class TransportStopDataFrameTransformActionTests extends ESTestCase {
         tasksBuilder.addTask(dataFrameIdBar,
                 DataFrameField.TASK_NAME, new DataFrameTransform(dataFrameIdBar),
                 new PersistentTasksCustomMetaData.Assignment("node-2", "test assignment"));
-        tasksBuilder.addTask(MlTasks.jobTaskId("foo-1"), MlTasks.JOB_TASK_NAME, new OpenJobAction.JobParams("foo-1"),
-                new PersistentTasksCustomMetaData.Assignment("node-3", "test assignment"));
+        tasksBuilder.addTask("test-task1", "testTasks", new PersistentTaskParams() {
+                @Override
+                public String getWriteableName() {
+                    return "testTasks";
+                }
+
+                @Override
+                public Version getMinimalSupportedVersion() {
+                    return null;
+                }
+
+                @Override
+                public void writeTo(StreamOutput out) throws IOException {
+
+                }
+
+                @Override
+                public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+                    return null;
+                }
+            },
+            new PersistentTasksCustomMetaData.Assignment("node-3", "test assignment"));
 
         ClusterState cs = ClusterState.builder(new ClusterName("_name"))
                 .metaData(MetaData.builder().putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build()))

x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_crud.yml

@@ -63,7 +63,8 @@ setup:
             "pivot": {
               "group_by": { "airline": {"terms": {"field": "airline"}}},
               "aggs": {"avg_response": {"avg": {"field": "responsetime"}}}
-            }
+            },
+            "description": "yaml test transform on airline-data"
           }
   - match: { acknowledged: true }
 
@@ -91,6 +92,7 @@ setup:
   - is_true: transforms.0.source.query.match_all
   - match: { transforms.0.pivot.group_by.airline.terms.field: "airline" }
   - match: { transforms.0.pivot.aggregations.avg_response.avg.field: "responsetime" }
+  - match: { transforms.0.description: "yaml test transform on airline-data" }
 
   - do:
       data_frame.get_data_frame_transform:
@@ -98,6 +100,7 @@ setup:
   - match: { count: 2 }
   - match: { transforms.0.id: "airline-transform" }
   - match: { transforms.1.id: "airline-transform-dos" }
+  - is_false: transforms.1.description
 
   - do:
       data_frame.get_data_frame_transform: