[ML] adding for_export flag for ml plugin GET resource APIs (#63092)

This adds the new `for_export` flag to the following APIs:

- GET _ml/anomaly_detection/<job_id>
- GET _ml/datafeeds/<datafeed_id>
- GET _ml/data_frame/analytics/<analytics_id>

The flag is designed for cloning or exporting configuration objects to later be put into the same cluster or a separate cluster. 

The following fields are not returned in the objects:

- any field that is not user settable (e.g. version, create_time)
- any field that is a calculated default value (e.g. datafeed chunking_config)
- any field that would effectively require changing to be of use (e.g. datafeed job_id)
- any field that is automatically set via another Elastic stack process (e.g. anomaly job custom_settings.created_by)


closes https://github.com/elastic/elasticsearch/issues/63055

client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java

@@ -121,7 +121,10 @@ final class MLRequestConverters {
 
         RequestConverters.Params params = new RequestConverters.Params();
         if (getJobRequest.getAllowNoMatch() != null) {
-            params.putParam("allow_no_match", Boolean.toString(getJobRequest.getAllowNoMatch()));
+            params.putParam(GetJobRequest.ALLOW_NO_MATCH.getPreferredName(), Boolean.toString(getJobRequest.getAllowNoMatch()));
+        }
+        if (getJobRequest.getForExport() != null) {
+            params.putParam(GetJobRequest.FOR_EXPORT, Boolean.toString(getJobRequest.getForExport()));
         }
         request.addParameters(params.asMap());
         return request;
@@ -270,6 +273,9 @@ final class MLRequestConverters {
             params.putParam(GetDatafeedRequest.ALLOW_NO_MATCH.getPreferredName(),
                     Boolean.toString(getDatafeedRequest.getAllowNoMatch()));
         }
+        if (getDatafeedRequest.getForExport() != null) {
+            params.putParam(GetDatafeedRequest.FOR_EXPORT, Boolean.toString(getDatafeedRequest.getForExport()));
+        }
         request.addParameters(params.asMap());
         return request;
     }
@@ -645,7 +651,10 @@ final class MLRequestConverters {
             }
         }
         if (getRequest.getAllowNoMatch() != null) {
-            params.putParam(GetDataFrameAnalyticsRequest.ALLOW_NO_MATCH.getPreferredName(), Boolean.toString(getRequest.getAllowNoMatch()));
+            params.putParam(GetDataFrameAnalyticsRequest.ALLOW_NO_MATCH, Boolean.toString(getRequest.getAllowNoMatch()));
+        }
+        if (getRequest.getForExport() != null) {
+            params.putParam(GetDataFrameAnalyticsRequest.FOR_EXPORT, Boolean.toString(getRequest.getForExport()));
         }
         request.addParameters(params.asMap());
         return request;

client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java

@@ -23,7 +23,6 @@ import org.elasticsearch.client.Validatable;
 import org.elasticsearch.client.ValidationException;
 import org.elasticsearch.client.core.PageParams;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.ParseField;
 
 import java.util.Arrays;
 import java.util.List;
@@ -32,11 +31,13 @@ import java.util.Optional;
 
 public class GetDataFrameAnalyticsRequest implements Validatable {
 
-    public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match");
+    public static final String ALLOW_NO_MATCH = "allow_no_match";
+    public static final String FOR_EXPORT = "for_export";
 
     private final List<String> ids;
     private Boolean allowNoMatch;
     private PageParams pageParams;
+    private Boolean forExport;
 
     /**
      * Helper method to create a request that will get ALL Data Frame Analytics
@@ -58,6 +59,22 @@ public class GetDataFrameAnalyticsRequest implements Validatable {
         return allowNoMatch;
     }
 
+    /**
+     * Setting this flag to `true` removes certain fields from the configuration on retrieval.
+     *
+     * This is useful when getting the configuration and wanting to put it in another cluster.
+     *
+     * Default value is false.
+     * @param forExport Boolean value indicating if certain fields should be removed
+     */
+    public void setForExport(boolean forExport) {
+        this.forExport = forExport;
+    }
+
+    public Boolean getForExport() {
+        return forExport;
+    }
+
     /**
      * Whether to ignore if a wildcard expression matches no data frame analytics.
      *
@@ -94,11 +111,12 @@ public class GetDataFrameAnalyticsRequest implements Validatable {
         GetDataFrameAnalyticsRequest other = (GetDataFrameAnalyticsRequest) o;
         return Objects.equals(ids, other.ids)
             && Objects.equals(allowNoMatch, other.allowNoMatch)
+            && Objects.equals(forExport, other.forExport)
             && Objects.equals(pageParams, other.pageParams);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(ids, allowNoMatch, pageParams);
+        return Objects.hash(ids, allowNoMatch, forExport, pageParams);
     }
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDatafeedRequest.java

@@ -41,10 +41,12 @@ public class GetDatafeedRequest implements Validatable, ToXContentObject {
 
     public static final ParseField DATAFEED_IDS = new ParseField("datafeed_ids");
     public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match");
+    public static final String FOR_EXPORT = "for_export";
 
     private static final String ALL_DATAFEEDS = "_all";
     private final List<String> datafeedIds;
     private Boolean allowNoMatch;
+    private Boolean forExport;
 
     @SuppressWarnings("unchecked")
     public static final ConstructingObjectParser<GetDatafeedRequest, Void> PARSER = new ConstructingObjectParser<>(
@@ -100,9 +102,25 @@ public class GetDatafeedRequest implements Validatable, ToXContentObject {
         return allowNoMatch;
     }
 
+    /**
+     * Setting this flag to `true` removes certain fields from the configuration on retrieval.
+     *
+     * This is useful when getting the configuration and wanting to put it in another cluster.
+     *
+     * Default value is false.
+     * @param forExport Boolean value indicating if certain fields should be removed
+     */
+    public void setForExport(boolean forExport) {
+        this.forExport = forExport;
+    }
+
+    public Boolean getForExport() {
+        return forExport;
+    }
+
     @Override
     public int hashCode() {
-        return Objects.hash(datafeedIds, allowNoMatch);
+        return Objects.hash(datafeedIds, forExport, allowNoMatch);
     }
 
     @Override
@@ -117,7 +135,8 @@ public class GetDatafeedRequest implements Validatable, ToXContentObject {
 
         GetDatafeedRequest that = (GetDatafeedRequest) other;
         return Objects.equals(datafeedIds, that.datafeedIds) &&
-            Objects.equals(allowNoMatch, that.allowNoMatch);
+            Objects.equals(allowNoMatch, that.allowNoMatch) &&
+            Objects.equals(forExport, that.forExport);
     }
 
     @Override

client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetJobRequest.java

@@ -42,10 +42,12 @@ public class GetJobRequest implements Validatable, ToXContentObject {
 
     public static final ParseField JOB_IDS = new ParseField("job_ids");
     public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match");
+    public static final String FOR_EXPORT = "for_export";
 
     private static final String ALL_JOBS = "_all";
     private final List<String> jobIds;
     private Boolean allowNoMatch;
+    private Boolean forExport;
 
     @SuppressWarnings("unchecked")
     public static final ConstructingObjectParser<GetJobRequest, Void> PARSER = new ConstructingObjectParser<>(
@@ -100,9 +102,25 @@ public class GetJobRequest implements Validatable, ToXContentObject {
         return allowNoMatch;
     }
 
+    /**
+     * Setting this flag to `true` removes certain fields from the configuration on retrieval.
+     *
+     * This is useful when getting the configuration and wanting to put it in another cluster.
+     *
+     * Default value is false.
+     * @param forExport Boolean value indicating if certain fields should be removed
+     */
+    public void setForExport(boolean forExport) {
+        this.forExport = forExport;
+    }
+
+    public Boolean getForExport() {
+        return forExport;
+    }
+
     @Override
     public int hashCode() {
-        return Objects.hash(jobIds, allowNoMatch);
+        return Objects.hash(jobIds, forExport, allowNoMatch);
     }
 
     @Override
@@ -117,6 +135,7 @@ public class GetJobRequest implements Validatable, ToXContentObject {
 
         GetJobRequest that = (GetJobRequest) other;
         return Objects.equals(jobIds, that.jobIds) &&
+            Objects.equals(forExport, that.forExport) &&
             Objects.equals(allowNoMatch, that.allowNoMatch);
     }
 

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java

@@ -340,6 +340,7 @@ public class MlClientDocumentationIT extends ESRestHighLevelClientTestCase {
             // tag::get-job-request
             GetJobRequest request = new GetJobRequest("get-machine-learning-job1", "get-machine-learning-job*"); // <1>
             request.setAllowNoMatch(true); // <2>
+            request.setForExport(false); // <3>
             // end::get-job-request
 
             // tag::get-job-execute
@@ -836,6 +837,7 @@ public class MlClientDocumentationIT extends ESRestHighLevelClientTestCase {
             // tag::get-datafeed-request
             GetDatafeedRequest request = new GetDatafeedRequest(datafeedId); // <1>
             request.setAllowNoMatch(true); // <2>
+            request.setForExport(false); // <3>
             // end::get-datafeed-request
 
             // tag::get-datafeed-execute
@@ -2861,6 +2863,7 @@ public class MlClientDocumentationIT extends ESRestHighLevelClientTestCase {
         {
             // tag::get-data-frame-analytics-request
             GetDataFrameAnalyticsRequest request = new GetDataFrameAnalyticsRequest("my-analytics-config"); // <1>
+            request.setForExport(false); // <2>
             // end::get-data-frame-analytics-request
 
             // tag::get-data-frame-analytics-execute

docs/java-rest/high-level/ml/get-data-frame-analytics.asciidoc

@@ -21,6 +21,9 @@ IDs, or the special wildcard `_all` to get all {dfanalytics-jobs}.
 include-tagged::{doc-tests-file}[{api}-request]
 --------------------------------------------------
 <1> Constructing a new GET request referencing an existing {dfanalytics-job}
+<2> Optional boolean value for requesting the {dfanalytics-job} in a format that can
+then be put into another cluster. Certain fields that can only be set when
+the {dfanalytics-job} is created are removed.
 
 include::../execution.asciidoc[]
 

docs/java-rest/high-level/ml/get-datafeed.asciidoc

@@ -25,6 +25,9 @@ include-tagged::{doc-tests-file}[{api}-request]
 contain wildcards.
 <2> Whether to ignore if a wildcard expression matches no datafeeds.
  (This includes `_all` string or when no datafeeds have been specified).
+<3> Optional boolean value for requesting the datafeed in a format that can
+then be put into another cluster. Certain fields that can only be set when
+the datafeed is created are removed.
 
 [id="{upid}-{api}-response"]
 ==== Get datafeed response

docs/java-rest/high-level/ml/get-job.asciidoc

@@ -25,6 +25,9 @@ include-tagged::{doc-tests-file}[{api}-request]
 wildcards.
 <2> Whether to ignore if a wildcard expression matches no {anomaly-jobs}.
  (This includes `_all` string or when no jobs have been specified).
+<3> Optional boolean value for requesting the job in a format that can
+then be put into another cluster. Certain fields that can only be set when
+the job is created are removed.
 
 [id="{upid}-{api}-response"]
 ==== Get {anomaly-jobs} response

docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc

@@ -57,6 +57,10 @@ all {dfeeds}.
 (Optional, boolean)
 include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-datafeeds]
 
+`for_export`::
+(Optional, boolean)
+include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=for-export]
+
 [[ml-get-datafeed-results]]
 == {api-response-body-title}
 

docs/reference/ml/anomaly-detection/apis/get-job.asciidoc

@@ -50,6 +50,9 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection-defaul
 (Optional, boolean)
 include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-jobs]
 
+`for_export`::
+(Optional, boolean)
+include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=for-export]
 
 [[ml-get-job-results]]
 == {api-response-body-title}

docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc

@@ -71,6 +71,10 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=from]
 (Optional, integer) 
 include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=size]
 
+`for_export`::
+(Optional, boolean)
+include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=for-export]
+
 [role="child_attributes"]
 [[ml-get-dfanalytics-results]]
 == {api-response-body-title}

docs/reference/ml/df-analytics/apis/get-trained-models.asciidoc

@@ -67,9 +67,7 @@ Specifies whether the included model definition should be returned as a JSON map
 
 `for_export`::
 (Optional, boolean)
-Indicates if certain fields should be removed from the model configuration on
-retrieval. This allows the model to be in an acceptable format to be retrieved
-and then added to another cluster. Default is false.
+include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=for-export]
 
 `from`::
 (Optional, integer)

docs/reference/ml/ml-shared.asciidoc

@@ -678,6 +678,12 @@ The number of individual forecasts currently available for the job. A value of
 `1` or more indicates that forecasts exist.
 end::forecast-total[]
 
+tag::for-export[]
+Indicates if certain fields should be removed from the configuration on
+retrieval. This allows the configuration to be in an acceptable format to be retrieved
+and then added to another cluster. Default is false.
+end::for-export[]
+
 tag::frequency[]
 The interval at which scheduled queries are made while the {dfeed} runs in real
 time. The default value is either the bucket span for short bucket spans, or,

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java

@@ -11,6 +11,7 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.AbstractDiffable;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -49,6 +50,8 @@ import java.util.Objects;
 import java.util.Random;
 import java.util.concurrent.TimeUnit;
 
+import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT;
+
 /**
  * Datafeed configuration options. Describes where to proactively pull input
  * data from.
@@ -65,6 +68,9 @@ public class DatafeedConfig extends AbstractDiffable<DatafeedConfig> implements
     private static final int TWO_MINS_SECONDS = 2 * SECONDS_IN_MINUTE;
     private static final int TWENTY_MINS_SECONDS = 20 * SECONDS_IN_MINUTE;
     private static final int HALF_DAY_SECONDS = 12 * 60 * SECONDS_IN_MINUTE;
+    public static final int DEFAULT_AGGREGATION_CHUNKING_BUCKETS = 1000;
+    private static final TimeValue MIN_DEFAULT_QUERY_DELAY = TimeValue.timeValueMinutes(1);
+    private static final TimeValue MAX_DEFAULT_QUERY_DELAY = TimeValue.timeValueMinutes(2);
 
     private static final Logger logger = LogManager.getLogger(DatafeedConfig.class);
 
@@ -451,17 +457,43 @@ public class DatafeedConfig extends AbstractDiffable<DatafeedConfig> implements
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
-        builder.field(ID.getPreferredName(), id);
-        builder.field(Job.ID.getPreferredName(), jobId);
-        if (params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) {
-            builder.field(CONFIG_TYPE.getPreferredName(), TYPE);
+        if (params.paramAsBoolean(FOR_EXPORT, false) == false) {
+            builder.field(ID.getPreferredName(), id);
+            // We don't include the job_id in export as we assume the PUT will be referring to a new job as well
+            builder.field(Job.ID.getPreferredName(), jobId);
+            if (params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) {
+                builder.field(CONFIG_TYPE.getPreferredName(), TYPE);
+            }
+            if (headers.isEmpty() == false && params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) {
+                builder.field(HEADERS.getPreferredName(), headers);
+            }
+            builder.field(QUERY_DELAY.getPreferredName(), queryDelay.getStringRep());
+            if (chunkingConfig != null) {
+                builder.field(CHUNKING_CONFIG.getPreferredName(), chunkingConfig);
+            }
+            builder.startObject(INDICES_OPTIONS.getPreferredName());
+            indicesOptions.toXContent(builder, params);
+            builder.endObject();
+        } else { // Don't include random defaults or unnecessary defaults in export
+            if (queryDelay.equals(defaultRandomQueryDelay(jobId)) == false) {
+                builder.field(QUERY_DELAY.getPreferredName(), queryDelay.getStringRep());
+            }
+            // Indices options are a pretty advanced feature, better to not include them if they are just the default ones
+            if (indicesOptions.equals(SearchRequest.DEFAULT_INDICES_OPTIONS) == false) {
+                builder.startObject(INDICES_OPTIONS.getPreferredName());
+                indicesOptions.toXContent(builder, params);
+                builder.endObject();
+            }
+            // Removing the default chunking config as it is determined by OTHER fields
+            if (chunkingConfig != null && chunkingConfig.equals(defaultChunkingConfig(aggProvider)) == false) {
+                builder.field(CHUNKING_CONFIG.getPreferredName(), chunkingConfig);
+            }
         }
-        builder.field(QUERY_DELAY.getPreferredName(), queryDelay.getStringRep());
+        builder.field(QUERY.getPreferredName(), queryProvider.getQuery());
         if (frequency != null) {
             builder.field(FREQUENCY.getPreferredName(), frequency.getStringRep());
         }
         builder.field(INDICES.getPreferredName(), indices);
-        builder.field(QUERY.getPreferredName(), queryProvider.getQuery());
         if (aggProvider != null) {
             builder.field(AGGREGATIONS.getPreferredName(), aggProvider.getAggs());
         }
@@ -473,26 +505,34 @@ public class DatafeedConfig extends AbstractDiffable<DatafeedConfig> implements
             builder.endObject();
         }
         builder.field(SCROLL_SIZE.getPreferredName(), scrollSize);
-        if (chunkingConfig != null) {
-            builder.field(CHUNKING_CONFIG.getPreferredName(), chunkingConfig);
-        }
-        if (headers.isEmpty() == false && params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) {
-            builder.field(HEADERS.getPreferredName(), headers);
-        }
         if (delayedDataCheckConfig != null) {
             builder.field(DELAYED_DATA_CHECK_CONFIG.getPreferredName(), delayedDataCheckConfig);
         }
         if (maxEmptySearches != null) {
             builder.field(MAX_EMPTY_SEARCHES.getPreferredName(), maxEmptySearches);
         }
-        builder.startObject(INDICES_OPTIONS.getPreferredName());
-        indicesOptions.toXContent(builder, params);
-        builder.endObject();
-
         builder.endObject();
         return builder;
     }
 
+    private static TimeValue defaultRandomQueryDelay(String jobId) {
+        Random random = new Random(jobId.hashCode());
+        long delayMillis = random.longs(MIN_DEFAULT_QUERY_DELAY.millis(), MAX_DEFAULT_QUERY_DELAY.millis()).findFirst().getAsLong();
+        return TimeValue.timeValueMillis(delayMillis);
+    }
+
+    private static ChunkingConfig defaultChunkingConfig(@Nullable AggProvider aggProvider) {
+        if (aggProvider == null || aggProvider.getParsedAggs() == null) {
+            return ChunkingConfig.newAuto();
+        } else {
+            long histogramIntervalMillis = ExtractorUtils.getHistogramIntervalMillis(aggProvider.getParsedAggs());
+            if (histogramIntervalMillis <= 0) {
+                throw ExceptionsHelper.badRequestException(Messages.DATAFEED_AGGREGATIONS_INTERVAL_MUST_BE_GREATER_THAN_ZERO);
+            }
+            return ChunkingConfig.newManual(TimeValue.timeValueMillis(DEFAULT_AGGREGATION_CHUNKING_BUCKETS * histogramIntervalMillis));
+        }
+    }
+
     /**
      * The lists of indices and types are compared for equality but they are not
      * sorted first so this test could fail simply because the indices and types
@@ -586,10 +626,6 @@ public class DatafeedConfig extends AbstractDiffable<DatafeedConfig> implements
 
     public static class Builder {
 
-        public static final int DEFAULT_AGGREGATION_CHUNKING_BUCKETS = 1000;
-        private static final TimeValue MIN_DEFAULT_QUERY_DELAY = TimeValue.timeValueMinutes(1);
-        private static final TimeValue MAX_DEFAULT_QUERY_DELAY = TimeValue.timeValueMinutes(2);
-
         private String id;
         private String jobId;
         private TimeValue queryDelay;
@@ -826,25 +862,13 @@ public class DatafeedConfig extends AbstractDiffable<DatafeedConfig> implements
 
         private void setDefaultChunkingConfig() {
             if (chunkingConfig == null) {
-                if (aggProvider == null || aggProvider.getParsedAggs() == null) {
-                    chunkingConfig = ChunkingConfig.newAuto();
-                } else {
-                    long histogramIntervalMillis = ExtractorUtils.getHistogramIntervalMillis(aggProvider.getParsedAggs());
-                    if (histogramIntervalMillis <= 0) {
-                        throw ExceptionsHelper.badRequestException(Messages.DATAFEED_AGGREGATIONS_INTERVAL_MUST_BE_GREATER_THAN_ZERO);
-                    }
-                    chunkingConfig = ChunkingConfig.newManual(TimeValue.timeValueMillis(
-                            DEFAULT_AGGREGATION_CHUNKING_BUCKETS * histogramIntervalMillis));
-                }
+                chunkingConfig = defaultChunkingConfig(aggProvider);
             }
         }
 
         private void setDefaultQueryDelay() {
             if (queryDelay == null) {
-                Random random = new Random(jobId.hashCode());
-                long delayMillis = random.longs(MIN_DEFAULT_QUERY_DELAY.millis(), MAX_DEFAULT_QUERY_DELAY.millis())
-                        .findFirst().getAsLong();
-                queryDelay = TimeValue.timeValueMillis(delayMillis);
+                queryDelay = defaultRandomQueryDelay(jobId);
             }
         }
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java

@@ -35,6 +35,7 @@ import java.util.Objects;
 
 import static org.elasticsearch.common.xcontent.ObjectParser.ValueType.OBJECT_ARRAY_BOOLEAN_OR_STRING;
 import static org.elasticsearch.common.xcontent.ObjectParser.ValueType.VALUE;
+import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT;
 
 public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable {
 
@@ -227,34 +228,34 @@ public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
-        builder.field(ID.getPreferredName(), id);
+        if (params.paramAsBoolean(FOR_EXPORT, false) == false) {
+            builder.field(ID.getPreferredName(), id);
+            if (createTime != null) {
+                builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.toEpochMilli());
+            }
+            if (version != null) {
+                builder.field(VERSION.getPreferredName(), version);
+            }
+            if (headers.isEmpty() == false && params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) {
+                builder.field(HEADERS.getPreferredName(), headers);
+            }
+            if (params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) {
+                builder.field(CONFIG_TYPE.getPreferredName(), TYPE);
+            }
+        }
         if (description != null) {
             builder.field(DESCRIPTION.getPreferredName(), description);
         }
         builder.field(SOURCE.getPreferredName(), source);
         builder.field(DEST.getPreferredName(), dest);
-
         builder.startObject(ANALYSIS.getPreferredName());
         builder.field(analysis.getWriteableName(), analysis,
             new MapParams(Collections.singletonMap(VERSION.getPreferredName(), version == null ? null : version.toString())));
         builder.endObject();
-
-        if (params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) {
-            builder.field(CONFIG_TYPE.getPreferredName(), TYPE);
-        }
         if (analyzedFields != null) {
             builder.field(ANALYZED_FIELDS.getPreferredName(), analyzedFields);
         }
         builder.field(MODEL_MEMORY_LIMIT.getPreferredName(), getModelMemoryLimit().getStringRep());
-        if (headers.isEmpty() == false && params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) {
-            builder.field(HEADERS.getPreferredName(), headers);
-        }
-        if (createTime != null) {
-            builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.toEpochMilli());
-        }
-        if (version != null) {
-            builder.field(VERSION.getPreferredName(), version);
-        }
         builder.field(ALLOW_LAZY_START.getPreferredName(), allowLazyStart);
         builder.field(MAX_NUM_THREADS.getPreferredName(), maxNumThreads);
         builder.endObject();

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java

@@ -45,6 +45,7 @@ import java.util.stream.Collectors;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 import static org.elasticsearch.xpack.core.ml.utils.NamedXContentObjectHelper.writeNamedObject;
+import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT;
 
 
 public class TrainedModelConfig implements ToXContentObject, Writeable {
@@ -52,7 +53,6 @@ public class TrainedModelConfig implements ToXContentObject, Writeable {
     public static final String NAME = "trained_model_config";
     public static final int CURRENT_DEFINITION_COMPRESSION_VERSION = 1;
     public static final String DECOMPRESS_DEFINITION = "decompress_definition";
-    public static final String FOR_EXPORT = "for_export";
     public static final String TOTAL_FEATURE_IMPORTANCE = "total_feature_importance";
     private static final Set<String> RESERVED_METADATA_FIELDS = Collections.singleton(TOTAL_FEATURE_IMPORTANCE);
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java

@@ -33,6 +33,7 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -42,6 +43,8 @@ import java.util.Set;
 import java.util.TreeSet;
 import java.util.concurrent.TimeUnit;
 
+import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT;
+
 /**
  * This class represents a configured and created Job. The creation time is set
  * to the time the object was constructed and the finished time and last
@@ -513,11 +516,35 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
 
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
         final String humanReadableSuffix = "_string";
-
-        builder.field(ID.getPreferredName(), jobId);
-        builder.field(JOB_TYPE.getPreferredName(), jobType);
-        if (jobVersion != null) {
-            builder.field(JOB_VERSION.getPreferredName(), jobVersion);
+        if (params.paramAsBoolean(FOR_EXPORT, false) == false) {
+            builder.field(ID.getPreferredName(), jobId);
+            builder.field(JOB_TYPE.getPreferredName(), jobType);
+            if (jobVersion != null) {
+                builder.field(JOB_VERSION.getPreferredName(), jobVersion);
+            }
+            builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + humanReadableSuffix, createTime.getTime());
+            if (finishedTime != null) {
+                builder.timeField(FINISHED_TIME.getPreferredName(), FINISHED_TIME.getPreferredName() + humanReadableSuffix,
+                    finishedTime.getTime());
+            }
+            if (modelSnapshotId != null) {
+                builder.field(MODEL_SNAPSHOT_ID.getPreferredName(), modelSnapshotId);
+            }
+            if (deleting) {
+                builder.field(DELETING.getPreferredName(), deleting);
+            }
+            if (modelSnapshotMinVersion != null) {
+                builder.field(MODEL_SNAPSHOT_MIN_VERSION.getPreferredName(), modelSnapshotMinVersion);
+            }
+            if (customSettings != null) {
+                builder.field(CUSTOM_SETTINGS.getPreferredName(), customSettings);
+            }
+        } else {
+            if (customSettings != null) {
+                HashMap<String, Object> newCustomSettings = new HashMap<>(customSettings);
+                newCustomSettings.remove("created_by");
+                builder.field(CUSTOM_SETTINGS.getPreferredName(), newCustomSettings);
+            }
         }
         if (groups.isEmpty() == false) {
             builder.field(GROUPS.getPreferredName(), groups);
@@ -525,11 +552,6 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
         if (description != null) {
             builder.field(DESCRIPTION.getPreferredName(), description);
         }
-        builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + humanReadableSuffix, createTime.getTime());
-        if (finishedTime != null) {
-            builder.timeField(FINISHED_TIME.getPreferredName(), FINISHED_TIME.getPreferredName() + humanReadableSuffix,
-                    finishedTime.getTime());
-        }
         builder.field(ANALYSIS_CONFIG.getPreferredName(), analysisConfig, params);
         if (analysisLimits != null) {
             builder.field(ANALYSIS_LIMITS.getPreferredName(), analysisLimits, params);
@@ -555,19 +577,7 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
         if (resultsRetentionDays != null) {
             builder.field(RESULTS_RETENTION_DAYS.getPreferredName(), resultsRetentionDays);
         }
-        if (customSettings != null) {
-            builder.field(CUSTOM_SETTINGS.getPreferredName(), customSettings);
-        }
-        if (modelSnapshotId != null) {
-            builder.field(MODEL_SNAPSHOT_ID.getPreferredName(), modelSnapshotId);
-        }
-        if (modelSnapshotMinVersion != null) {
-            builder.field(MODEL_SNAPSHOT_MIN_VERSION.getPreferredName(), modelSnapshotMinVersion);
-        }
         builder.field(RESULTS_INDEX_NAME.getPreferredName(), resultsIndexName);
-        if (deleting) {
-            builder.field(DELETING.getPreferredName(), deleting);
-        }
         builder.field(ALLOW_LAZY_OPEN.getPreferredName(), allowLazyOpen);
         return builder;
     }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ToXContentParams.java

@@ -18,6 +18,14 @@ public final class ToXContentParams {
      */
     public static final String FOR_INTERNAL_STORAGE = "for_internal_storage";
 
+    /**
+     * Parameter to indicate if this XContent serialization should only include fields that are allowed to be used
+     * on PUT
+     *
+     * This helps to GET a configuration, copy it, and then PUT it directly without removing or changing any fields in between
+     */
+    public static final String FOR_EXPORT = "for_export";
+
     /**
      * When serialising POJOs to X Content this indicates whether the calculated (i.e. not stored) fields
      * should be included or not

x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java

@@ -18,6 +18,7 @@ import org.yaml.snakeyaml.util.UriEncoder;
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -106,31 +107,7 @@ public class MlBasicMultiNodeIT extends ESRestTestCase {
     }
 
     public void testMiniFarequoteWithDatafeeder() throws Exception {
-        boolean datesHaveNanoSecondResolution = randomBoolean();
-        String dateMappingType = datesHaveNanoSecondResolution ? "date_nanos" : "date";
-        String dateFormat = datesHaveNanoSecondResolution ? "strict_date_optional_time_nanos" : "strict_date_optional_time";
-        String randomNanos = datesHaveNanoSecondResolution ? "," + randomIntBetween(100000000, 999999999) : "";
-        Request createAirlineDataRequest = new Request("PUT", "/airline-data");
-        createAirlineDataRequest.setJsonEntity("{"
-                + "  \"mappings\": {"
-                + "    \"properties\": {"
-                + "      \"time\": { \"type\":\"" + dateMappingType + "\", \"format\":\"" + dateFormat + "\"},"
-                + "      \"airline\": { \"type\":\"keyword\"},"
-                + "      \"responsetime\": { \"type\":\"float\"}"
-                + "    }"
-                + "  }"
-                + "}");
-        client().performRequest(createAirlineDataRequest);
-        Request airlineData1 = new Request("PUT", "/airline-data/_doc/1");
-        airlineData1.setJsonEntity("{\"time\":\"2016-06-01T00:00:00" + randomNanos + "Z\",\"airline\":\"AAA\",\"responsetime\":135.22}");
-        client().performRequest(airlineData1);
-        Request airlineData2 = new Request("PUT", "/airline-data/_doc/2");
-        airlineData2.setJsonEntity("{\"time\":\"2016-06-01T01:59:00" + randomNanos + "Z\",\"airline\":\"AAA\",\"responsetime\":541.76}");
-        client().performRequest(airlineData2);
-
-        // Ensure all data is searchable
-        refreshAllIndices();
-
+        createAndIndexFarequote();
         String jobId = "mini-farequote-with-data-feeder-job";
         createFarequoteJob(jobId);
         String datafeedId = "bar";
@@ -265,6 +242,211 @@ public class MlBasicMultiNodeIT extends ESRestTestCase {
         client().performRequest(new Request("DELETE", BASE_PATH + "anomaly_detectors/" + jobId));
     }
 
+    @SuppressWarnings("unchecked")
+    public void testExportAndPutJob() throws Exception {
+        String jobId = "test-export-import-job";
+        createFarequoteJob(jobId);
+        Response jobResponse = client().performRequest(
+            new Request("GET", BASE_PATH + "anomaly_detectors/" + jobId + "?for_export=true"));
+        Map<String, Object> originalJobBody = (Map<String, Object>)((List<?>) entityAsMap(jobResponse).get("jobs")).get(0);
+
+        XContentBuilder xContentBuilder = jsonBuilder().map(originalJobBody);
+        Request request = new Request("PUT", BASE_PATH + "anomaly_detectors/" + jobId + "-import");
+        request.setJsonEntity(Strings.toString(xContentBuilder));
+        client().performRequest(request);
+
+        Response importedJobResponse = client().performRequest(
+            new Request("GET", BASE_PATH + "anomaly_detectors/" + jobId + "-import" + "?for_export=true"));
+        Map<String, Object> importedJobBody = (Map<String, Object>)((List<?>) entityAsMap(importedJobResponse).get("jobs")).get(0);
+        assertThat(originalJobBody, equalTo(importedJobBody));
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testExportAndPutDatafeed() throws Exception {
+        createAndIndexFarequote();
+        String jobId = "test-export-import-datafeed";
+        createFarequoteJob(jobId);
+        String datafeedId = jobId + "-datafeed";
+        createDatafeed(datafeedId, jobId);
+
+        Response dfResponse = client().performRequest(
+            new Request("GET", BASE_PATH + "datafeeds/" + datafeedId + "?for_export=true"));
+        Map<String, Object> originalDfBody = (Map<String, Object>)((List<?>) entityAsMap(dfResponse).get("datafeeds")).get(0);
+
+        //Delete this so we can PUT another datafeed for the same job
+        client().performRequest(new Request("DELETE", BASE_PATH + "datafeeds/" + datafeedId));
+
+        Map<String, Object> toPut = new HashMap<>(originalDfBody);
+        toPut.put("job_id", jobId);
+        XContentBuilder xContentBuilder = jsonBuilder().map(toPut);
+        Request request = new Request("PUT", BASE_PATH + "datafeeds/" + datafeedId + "-import");
+        request.setJsonEntity(Strings.toString(xContentBuilder));
+        client().performRequest(request);
+
+        Response importedDfResponse = client().performRequest(
+            new Request("GET", BASE_PATH + "datafeeds/" + datafeedId + "-import" + "?for_export=true"));
+        Map<String, Object> importedDfBody = (Map<String, Object>)((List<?>) entityAsMap(importedDfResponse).get("datafeeds")).get(0);
+        assertThat(originalDfBody, equalTo(importedDfBody));
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testExportAndPutDataFrameAnalytics_OutlierDetection() throws Exception {
+        createAndIndexFarequote();
+        String analyticsId = "outlier-export-import";
+        XContentBuilder xContentBuilder = jsonBuilder();
+        xContentBuilder.startObject();
+        {
+            xContentBuilder.field("description", "outlier analytics");
+
+            xContentBuilder.startObject("source");
+            {
+                xContentBuilder.field("index", "airline-data");
+            }
+            xContentBuilder.endObject();
+            xContentBuilder.startObject("dest");
+            {
+                xContentBuilder.field("index", "outliers-airline-data");
+            }
+            xContentBuilder.endObject();
+            xContentBuilder.startObject("analysis");
+            {
+                xContentBuilder.startObject("outlier_detection");
+                {
+                    xContentBuilder.field("compute_feature_influence", false);
+                }
+                xContentBuilder.endObject();
+            }
+            xContentBuilder.endObject();
+        }
+        xContentBuilder.endObject();
+
+        Request request = new Request("PUT", BASE_PATH + "data_frame/analytics/" + analyticsId);
+        request.setJsonEntity(Strings.toString(xContentBuilder));
+        client().performRequest(request);
+
+        Response jobResponse = client().performRequest(
+            new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "?for_export=true"));
+        Map<String, Object> originalJobBody = (Map<String, Object>)((List<?>) entityAsMap(jobResponse).get("data_frame_analytics")).get(0);
+
+        XContentBuilder newBuilder = jsonBuilder().map(originalJobBody);
+        request = new Request("PUT", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import");
+        request.setJsonEntity(Strings.toString(newBuilder));
+        client().performRequest(request);
+
+        Response importedJobResponse = client().performRequest(
+            new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import" + "?for_export=true"));
+        Map<String, Object> importedJobBody = (Map<String, Object>)((List<?>) entityAsMap(importedJobResponse)
+            .get("data_frame_analytics"))
+            .get(0);
+        assertThat(originalJobBody, equalTo(importedJobBody));
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testExportAndPutDataFrameAnalytics_Regression() throws Exception {
+        createAndIndexFarequote();
+        String analyticsId = "regression-export-import";
+        XContentBuilder xContentBuilder = jsonBuilder();
+        xContentBuilder.startObject();
+        {
+            xContentBuilder.field("description", "regression analytics");
+
+            xContentBuilder.startObject("source");
+            {
+                xContentBuilder.field("index", "airline-data");
+            }
+            xContentBuilder.endObject();
+            xContentBuilder.startObject("dest");
+            {
+                xContentBuilder.field("index", "regression-airline-data");
+            }
+            xContentBuilder.endObject();
+            xContentBuilder.startObject("analysis");
+            {
+                xContentBuilder.startObject("regression");
+                {
+                    xContentBuilder.field("dependent_variable", "responsetime");
+                    xContentBuilder.field("training_percent", 50);
+                }
+                xContentBuilder.endObject();
+            }
+            xContentBuilder.endObject();
+        }
+        xContentBuilder.endObject();
+
+        Request request = new Request("PUT", BASE_PATH + "data_frame/analytics/" + analyticsId);
+        request.setJsonEntity(Strings.toString(xContentBuilder));
+        client().performRequest(request);
+
+        Response jobResponse = client().performRequest(
+            new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "?for_export=true"));
+        Map<String, Object> originalJobBody = (Map<String, Object>)((List<?>) entityAsMap(jobResponse).get("data_frame_analytics")).get(0);
+
+        XContentBuilder newBuilder = jsonBuilder().map(originalJobBody);
+        request = new Request("PUT", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import");
+        request.setJsonEntity(Strings.toString(newBuilder));
+        client().performRequest(request);
+
+        Response importedJobResponse = client().performRequest(
+            new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import" + "?for_export=true"));
+        Map<String, Object> importedJobBody = (Map<String, Object>)((List<?>) entityAsMap(importedJobResponse)
+            .get("data_frame_analytics"))
+            .get(0);
+        assertThat(originalJobBody, equalTo(importedJobBody));
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testExportAndPutDataFrameAnalytics_Classification() throws Exception {
+        createAndIndexFarequote();
+        String analyticsId = "classification-export-import";
+        XContentBuilder xContentBuilder = jsonBuilder();
+        xContentBuilder.startObject();
+        {
+            xContentBuilder.field("description", "classification analytics");
+
+            xContentBuilder.startObject("source");
+            {
+                xContentBuilder.field("index", "airline-data");
+            }
+            xContentBuilder.endObject();
+            xContentBuilder.startObject("dest");
+            {
+                xContentBuilder.field("index", "classification-airline-data");
+            }
+            xContentBuilder.endObject();
+            xContentBuilder.startObject("analysis");
+            {
+                xContentBuilder.startObject("classification");
+                {
+                    xContentBuilder.field("dependent_variable", "airline");
+                    xContentBuilder.field("training_percent", 60);
+                }
+                xContentBuilder.endObject();
+            }
+            xContentBuilder.endObject();
+        }
+        xContentBuilder.endObject();
+
+        Request request = new Request("PUT", BASE_PATH + "data_frame/analytics/" + analyticsId);
+        request.setJsonEntity(Strings.toString(xContentBuilder));
+        client().performRequest(request);
+
+        Response jobResponse = client().performRequest(
+            new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "?for_export=true"));
+        Map<String, Object> originalJobBody = (Map<String, Object>)((List<?>) entityAsMap(jobResponse).get("data_frame_analytics")).get(0);
+
+        XContentBuilder newBuilder = jsonBuilder().map(originalJobBody);
+        request = new Request("PUT", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import");
+        request.setJsonEntity(Strings.toString(newBuilder));
+        client().performRequest(request);
+
+        Response importedJobResponse = client().performRequest(
+            new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import" + "?for_export=true"));
+        Map<String, Object> importedJobBody = (Map<String, Object>)((List<?>) entityAsMap(importedJobResponse)
+            .get("data_frame_analytics"))
+            .get(0);
+        assertThat(originalJobBody, equalTo(importedJobBody));
+    }
+
     private Response createDatafeed(String datafeedId, String jobId) throws Exception {
         XContentBuilder xContentBuilder = jsonBuilder();
         xContentBuilder.startObject();
@@ -322,4 +504,31 @@ public class MlBasicMultiNodeIT extends ESRestTestCase {
         assertThat(asMap.get("flushed"), is(true));
         assertThat(asMap.get("last_finalized_bucket_end"), equalTo(expectedLastFinalizedBucketEnd));
     }
+
+    private void createAndIndexFarequote() throws Exception {
+        boolean datesHaveNanoSecondResolution = randomBoolean();
+        String dateMappingType = datesHaveNanoSecondResolution ? "date_nanos" : "date";
+        String dateFormat = datesHaveNanoSecondResolution ? "strict_date_optional_time_nanos" : "strict_date_optional_time";
+        String randomNanos = datesHaveNanoSecondResolution ? "," + randomIntBetween(100000000, 999999999) : "";
+        Request createAirlineDataRequest = new Request("PUT", "/airline-data");
+        createAirlineDataRequest.setJsonEntity("{"
+            + "  \"mappings\": {"
+            + "    \"properties\": {"
+            + "      \"time\": { \"type\":\"" + dateMappingType + "\", \"format\":\"" + dateFormat + "\"},"
+            + "      \"airline\": { \"type\":\"keyword\"},"
+            + "      \"responsetime\": { \"type\":\"float\"}"
+            + "    }"
+            + "  }"
+            + "}");
+        client().performRequest(createAirlineDataRequest);
+        Request airlineData1 = new Request("PUT", "/airline-data/_doc/1");
+        airlineData1.setJsonEntity("{\"time\":\"2016-06-01T00:00:00" + randomNanos + "Z\",\"airline\":\"AAA\",\"responsetime\":135.22}");
+        client().performRequest(airlineData1);
+        Request airlineData2 = new Request("PUT", "/airline-data/_doc/2");
+        airlineData2.setJsonEntity("{\"time\":\"2016-06-01T01:59:00" + randomNanos + "Z\",\"airline\":\"AAA\",\"responsetime\":541.76}");
+        client().performRequest(airlineData2);
+
+        // Ensure all data is searchable
+        refreshAllIndices();
+    }
 }

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractor.java

@@ -320,11 +320,11 @@ public class ChunkedDataExtractor implements DataExtractor {
         }
 
         /**
-         * This heuristic is a direct copy of the manual chunking config auto-creation done in {@link DatafeedConfig.Builder}
+         * This heuristic is a direct copy of the manual chunking config auto-creation done in {@link DatafeedConfig}
          */
         @Override
         public long estimateChunk() {
-            return DatafeedConfig.Builder.DEFAULT_AGGREGATION_CHUNKING_BUCKETS * histogramIntervalMillis;
+            return DatafeedConfig.DEFAULT_AGGREGATION_CHUNKING_BUCKETS * histogramIntervalMillis;
         }
 
         @Override

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java

@@ -16,9 +16,12 @@ import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
 import org.elasticsearch.xpack.ml.MachineLearning;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.List;
+import java.util.Set;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
+import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT;
 
 public class RestGetDatafeedsAction extends BaseRestHandler {
 
@@ -51,4 +54,9 @@ public class RestGetDatafeedsAction extends BaseRestHandler {
                 restRequest.paramAsBoolean(Request.ALLOW_NO_DATAFEEDS, request.allowNoMatch())));
         return channel -> client.execute(GetDatafeedsAction.INSTANCE, request, new RestToXContentListener<>(channel));
     }
+
+    @Override
+    protected Set<String> responseParams() {
+        return Collections.singleton(FOR_EXPORT);
+    }
 }

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsAction.java

@@ -16,9 +16,12 @@ import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
 import org.elasticsearch.xpack.ml.MachineLearning;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.List;
+import java.util.Set;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
+import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT;
 
 public class RestGetDataFrameAnalyticsAction extends BaseRestHandler {
 
@@ -49,4 +52,9 @@ public class RestGetDataFrameAnalyticsAction extends BaseRestHandler {
                 request.isAllowNoResources()));
         return channel -> client.execute(GetDataFrameAnalyticsAction.INSTANCE, request, new RestToXContentListener<>(channel));
     }
+
+    @Override
+    protected Set<String> responseParams() {
+        return Collections.singleton(FOR_EXPORT);
+    }
 }

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsAction.java

@@ -34,6 +34,7 @@ import java.util.Set;
 import static java.util.Arrays.asList;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.xpack.core.ml.action.GetTrainedModelsAction.Request.ALLOW_NO_MATCH;
+import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT;
 
 public class RestGetTrainedModelsAction extends BaseRestHandler {
 
@@ -98,7 +99,7 @@ public class RestGetTrainedModelsAction extends BaseRestHandler {
 
     @Override
     protected Set<String> responseParams() {
-        return Set.of(TrainedModelConfig.DECOMPRESS_DEFINITION, TrainedModelConfig.FOR_EXPORT);
+        return Set.of(TrainedModelConfig.DECOMPRESS_DEFINITION, FOR_EXPORT);
     }
 
     private static class RestToXContentListenerWithDefaultValues<T extends ToXContentObject> extends RestToXContentListener<T> {

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java

@@ -18,9 +18,12 @@ import org.elasticsearch.xpack.core.ml.job.config.Job;
 import org.elasticsearch.xpack.ml.MachineLearning;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.List;
+import java.util.Set;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
+import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT;
 
 public class RestGetJobsAction extends BaseRestHandler {
 
@@ -53,4 +56,9 @@ public class RestGetJobsAction extends BaseRestHandler {
                 restRequest.paramAsBoolean(Request.ALLOW_NO_JOBS, request.allowNoMatch())));
         return channel -> client.execute(GetJobsAction.INSTANCE, request, new RestToXContentListener<>(channel));
     }
+
+    @Override
+    protected Set<String> responseParams() {
+        return Collections.singleton(FOR_EXPORT);
+    }
 }

x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics.json

@@ -43,6 +43,12 @@
         "type":"int",
         "description":"specifies a max number of analytics to get",
         "default":100
+      },
+      "for_export": {
+        "required": false,
+        "type": "boolean",
+        "default": false,
+        "description": "Omits fields that are illegal to set on data frame analytics PUT"
       }
     }
   }

x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeeds.json

@@ -38,6 +38,12 @@
         "required":false,
         "description":"Whether to ignore if a wildcard expression matches no datafeeds. (This includes `_all` string or when no datafeeds have been specified)",
         "deprecated":true
+      },
+      "for_export": {
+        "required": false,
+        "type": "boolean",
+        "default": false,
+        "description": "Omits fields that are illegal to set on datafeed PUT"
       }
     }
   }

x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_jobs.json

@@ -38,6 +38,12 @@
         "required":false,
         "description":"Whether to ignore if a wildcard expression matches no jobs. (This includes `_all` string or when no jobs have been specified)",
         "deprecated":true
+      },
+      "for_export": {
+        "required": false,
+        "type": "boolean",
+        "default": false,
+        "description": "Omits fields that are illegal to set on job PUT"
       }
     }
   }

x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml

@@ -2168,3 +2168,37 @@ setup:
           {
             "description": "blah"
           }
+
+---
+"Test GET config for export":
+
+  - do:
+      ml.put_data_frame_analytics:
+        id: "simple-outlier-detection"
+        body: >
+          {
+            "source": {
+              "index": "index-source"
+            },
+            "dest": {
+              "index": "index-dest"
+            },
+            "analysis": {"outlier_detection":{}}
+          }
+  - do:
+      ml.get_data_frame_analytics:
+        id: "simple-outlier-detection"
+        for_export: true
+  - match: { data_frame_analytics.0.source.index.0: "index-source" }
+  - match: { data_frame_analytics.0.source.query: {"match_all" : {} } }
+  - match: { data_frame_analytics.0.dest.index: "index-dest" }
+  - match: { data_frame_analytics.0.analysis: {
+    "outlier_detection":{
+      "compute_feature_influence": true,
+      "outlier_fraction": 0.05,
+      "standardization_enabled": true
+    }
+  }}
+  - is_false: data_frame_analytics.0.create_time
+  - is_false: data_frame_analytics.0.version
+  - is_false: data_frame_analytics.0.id

x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml

@@ -506,3 +506,23 @@ setup:
 
   - match: { datafeeds.0.indices_options.ignore_throttled: false }
   - match: { datafeeds.0.indices_options.allow_no_indices: false }
+---
+"Test get datafeed for export":
+  - do:
+      ml.put_datafeed:
+        datafeed_id: test-for-export
+        body:  >
+          {
+            "job_id":"datafeeds-crud-1",
+            "indexes":["index-foo"]
+          }
+  - do:
+      ml.get_datafeeds:
+        datafeed_id: test-for-export
+        for_export: true
+  - match: { datafeeds.0.indices.0: "index-foo"}
+  - is_false: datafeeds.0.datafeed_id
+  - is_false: datafeeds.0.job_id
+  - is_false: datafeeds.0.create_time
+  - is_false: datafeeds.0.query_delay
+  - is_false: datafeeds.0.chunking_config

x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get.yml

@@ -86,3 +86,19 @@ setup:
   - match: { jobs.0.description: "Job 1"}
   - match: { jobs.1.job_id: "jobs-get-2"}
   - match: { jobs.1.description: "Job 2"}
+---
+"Test get job for export":
+
+  - do:
+      ml.get_jobs:
+        job_id: jobs-get-1
+        for_export: true
+  - match: { jobs.0.description: "Job 1"}
+  - is_false: job_id
+  - is_false: job_type
+  - is_false: job_version
+  - is_false: create_time
+  - is_false: finished_time
+  - is_false: model_snapshot_id
+  - is_false: model_snapshot_min_version
+  - is_false: deleting