Adding `Job` and `AnalysisConfig` for HLRC (#32687)

* Adding `Job` and `AnalysisConfig` for HLRC

* Removing println used for local debugging

* Adding null checks and removing unnecessary field

x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/AnalysisConfig.java

@@ -0,0 +1,400 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.config;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.Function;
+
+/**
+ * Analysis configuration options that describe which fields are
+ * analyzed and which functions are used to detect anomalies.
+ * <p>
+ * The configuration can contain multiple detectors, a new anomaly detector will
+ * be created for each detector configuration. The fields
+ * <code>bucketSpan, summaryCountFieldName and categorizationFieldName</code>
+ * apply to all detectors.
+ * <p>
+ * If a value has not been set it will be <code>null</code>
+ * Object wrappers are used around integral types &amp; booleans so they can take
+ * <code>null</code> values.
+ */
+public class AnalysisConfig implements ToXContentObject {
+    /**
+     * Serialisation names
+     */
+    public static final ParseField ANALYSIS_CONFIG = new ParseField("analysis_config");
+    public static final ParseField BUCKET_SPAN = new ParseField("bucket_span");
+    public static final ParseField CATEGORIZATION_FIELD_NAME = new ParseField("categorization_field_name");
+    public static final ParseField CATEGORIZATION_FILTERS = new ParseField("categorization_filters");
+    public static final ParseField CATEGORIZATION_ANALYZER = CategorizationAnalyzerConfig.CATEGORIZATION_ANALYZER;
+    public static final ParseField LATENCY = new ParseField("latency");
+    public static final ParseField SUMMARY_COUNT_FIELD_NAME = new ParseField("summary_count_field_name");
+    public static final ParseField DETECTORS = new ParseField("detectors");
+    public static final ParseField INFLUENCERS = new ParseField("influencers");
+    public static final ParseField OVERLAPPING_BUCKETS = new ParseField("overlapping_buckets");
+    public static final ParseField RESULT_FINALIZATION_WINDOW = new ParseField("result_finalization_window");
+    public static final ParseField MULTIVARIATE_BY_FIELDS = new ParseField("multivariate_by_fields");
+
+    @SuppressWarnings("unchecked")
+    public static final ConstructingObjectParser<Builder, Void> PARSER = new ConstructingObjectParser<>(ANALYSIS_CONFIG.getPreferredName(),
+        true, a -> new AnalysisConfig.Builder((List<Detector>) a[0]));
+
+    static {
+        PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(),
+            (p, c) -> (Detector.PARSER).apply(p, c).build(), DETECTORS);
+        PARSER.declareString((builder, val) ->
+            builder.setBucketSpan(TimeValue.parseTimeValue(val, BUCKET_SPAN.getPreferredName())), BUCKET_SPAN);
+        PARSER.declareString(Builder::setCategorizationFieldName, CATEGORIZATION_FIELD_NAME);
+        PARSER.declareStringArray(Builder::setCategorizationFilters, CATEGORIZATION_FILTERS);
+        // This one is nasty - the syntax for analyzers takes either names or objects at many levels, hence it's not
+        // possible to simply declare whether the field is a string or object and a completely custom parser is required
+        PARSER.declareField(Builder::setCategorizationAnalyzerConfig,
+            (p, c) -> CategorizationAnalyzerConfig.buildFromXContentFragment(p),
+            CATEGORIZATION_ANALYZER, ObjectParser.ValueType.OBJECT_OR_STRING);
+        PARSER.declareString((builder, val) ->
+            builder.setLatency(TimeValue.parseTimeValue(val, LATENCY.getPreferredName())), LATENCY);
+        PARSER.declareString(Builder::setSummaryCountFieldName, SUMMARY_COUNT_FIELD_NAME);
+        PARSER.declareStringArray(Builder::setInfluencers, INFLUENCERS);
+        PARSER.declareBoolean(Builder::setOverlappingBuckets, OVERLAPPING_BUCKETS);
+        PARSER.declareLong(Builder::setResultFinalizationWindow, RESULT_FINALIZATION_WINDOW);
+        PARSER.declareBoolean(Builder::setMultivariateByFields, MULTIVARIATE_BY_FIELDS);
+    }
+
+    /**
+     * These values apply to all detectors
+     */
+    private final TimeValue bucketSpan;
+    private final String categorizationFieldName;
+    private final List<String> categorizationFilters;
+    private final CategorizationAnalyzerConfig categorizationAnalyzerConfig;
+    private final TimeValue latency;
+    private final String summaryCountFieldName;
+    private final List<Detector> detectors;
+    private final List<String> influencers;
+    private final Boolean overlappingBuckets;
+    private final Long resultFinalizationWindow;
+    private final Boolean multivariateByFields;
+
+    private AnalysisConfig(TimeValue bucketSpan, String categorizationFieldName, List<String> categorizationFilters,
+                           CategorizationAnalyzerConfig categorizationAnalyzerConfig, TimeValue latency, String summaryCountFieldName,
+                           List<Detector> detectors, List<String> influencers, Boolean overlappingBuckets, Long resultFinalizationWindow,
+                           Boolean multivariateByFields) {
+        this.detectors = Collections.unmodifiableList(detectors);
+        this.bucketSpan = bucketSpan;
+        this.latency = latency;
+        this.categorizationFieldName = categorizationFieldName;
+        this.categorizationAnalyzerConfig = categorizationAnalyzerConfig;
+        this.categorizationFilters = categorizationFilters == null ? null : Collections.unmodifiableList(categorizationFilters);
+        this.summaryCountFieldName = summaryCountFieldName;
+        this.influencers = Collections.unmodifiableList(influencers);
+        this.overlappingBuckets = overlappingBuckets;
+        this.resultFinalizationWindow = resultFinalizationWindow;
+        this.multivariateByFields = multivariateByFields;
+    }
+
+    /**
+     * The analysis bucket span
+     *
+     * @return The bucketspan or <code>null</code> if not set
+     */
+    public TimeValue getBucketSpan() {
+        return bucketSpan;
+    }
+
+    public String getCategorizationFieldName() {
+        return categorizationFieldName;
+    }
+
+    public List<String> getCategorizationFilters() {
+        return categorizationFilters;
+    }
+
+    public CategorizationAnalyzerConfig getCategorizationAnalyzerConfig() {
+        return categorizationAnalyzerConfig;
+    }
+
+    /**
+     * The latency interval during which out-of-order records should be handled.
+     *
+     * @return The latency interval or <code>null</code> if not set
+     */
+    public TimeValue getLatency() {
+        return latency;
+    }
+
+    /**
+     * The name of the field that contains counts for pre-summarised input
+     *
+     * @return The field name or <code>null</code> if not set
+     */
+    public String getSummaryCountFieldName() {
+        return summaryCountFieldName;
+    }
+
+    /**
+     * The list of analysis detectors. In a valid configuration the list should
+     * contain at least 1 {@link Detector}
+     *
+     * @return The Detectors used in this job
+     */
+    public List<Detector> getDetectors() {
+        return detectors;
+    }
+
+    /**
+     * The list of influence field names
+     */
+    public List<String> getInfluencers() {
+        return influencers;
+    }
+
+    public Boolean getOverlappingBuckets() {
+        return overlappingBuckets;
+    }
+
+    public Long getResultFinalizationWindow() {
+        return resultFinalizationWindow;
+    }
+
+    public Boolean getMultivariateByFields() {
+        return multivariateByFields;
+    }
+
+    private static void addIfNotNull(Set<String> fields, String field) {
+        if (field != null) {
+            fields.add(field);
+        }
+    }
+
+    public List<String> fields() {
+        return collectNonNullAndNonEmptyDetectorFields(Detector::getFieldName);
+    }
+
+    private List<String> collectNonNullAndNonEmptyDetectorFields(
+        Function<Detector, String> fieldGetter) {
+        Set<String> fields = new HashSet<>();
+
+        for (Detector d : getDetectors()) {
+            addIfNotNull(fields, fieldGetter.apply(d));
+        }
+
+        // remove empty strings
+        fields.remove("");
+
+        return new ArrayList<>(fields);
+    }
+
+    public List<String> byFields() {
+        return collectNonNullAndNonEmptyDetectorFields(Detector::getByFieldName);
+    }
+
+    public List<String> overFields() {
+        return collectNonNullAndNonEmptyDetectorFields(Detector::getOverFieldName);
+    }
+
+    public List<String> partitionFields() {
+        return collectNonNullAndNonEmptyDetectorFields(Detector::getPartitionFieldName);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        if (bucketSpan != null) {
+            builder.field(BUCKET_SPAN.getPreferredName(), bucketSpan.getStringRep());
+        }
+        if (categorizationFieldName != null) {
+            builder.field(CATEGORIZATION_FIELD_NAME.getPreferredName(), categorizationFieldName);
+        }
+        if (categorizationFilters != null) {
+            builder.field(CATEGORIZATION_FILTERS.getPreferredName(), categorizationFilters);
+        }
+        if (categorizationAnalyzerConfig != null) {
+            // This cannot be builder.field(CATEGORIZATION_ANALYZER.getPreferredName(), categorizationAnalyzerConfig, params);
+            // because that always writes categorizationAnalyzerConfig as an object, and in the case of a global analyzer it
+            // gets written as a single string.
+            categorizationAnalyzerConfig.toXContent(builder, params);
+        }
+        if (latency != null) {
+            builder.field(LATENCY.getPreferredName(), latency.getStringRep());
+        }
+        if (summaryCountFieldName != null) {
+            builder.field(SUMMARY_COUNT_FIELD_NAME.getPreferredName(), summaryCountFieldName);
+        }
+        builder.startArray(DETECTORS.getPreferredName());
+        for (Detector detector : detectors) {
+            detector.toXContent(builder, params);
+        }
+        builder.endArray();
+        builder.field(INFLUENCERS.getPreferredName(), influencers);
+        if (overlappingBuckets != null) {
+            builder.field(OVERLAPPING_BUCKETS.getPreferredName(), overlappingBuckets);
+        }
+        if (resultFinalizationWindow != null) {
+            builder.field(RESULT_FINALIZATION_WINDOW.getPreferredName(), resultFinalizationWindow);
+        }
+        if (multivariateByFields != null) {
+            builder.field(MULTIVARIATE_BY_FIELDS.getPreferredName(), multivariateByFields);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object object) {
+        if (this == object) {
+            return true;
+        }
+
+        if (object == null || getClass() != object.getClass()) {
+            return false;
+        }
+
+        AnalysisConfig that = (AnalysisConfig) object;
+        return Objects.equals(latency, that.latency) &&
+            Objects.equals(bucketSpan, that.bucketSpan) &&
+            Objects.equals(categorizationFieldName, that.categorizationFieldName) &&
+            Objects.equals(categorizationFilters, that.categorizationFilters) &&
+            Objects.equals(categorizationAnalyzerConfig, that.categorizationAnalyzerConfig) &&
+            Objects.equals(summaryCountFieldName, that.summaryCountFieldName) &&
+            Objects.equals(detectors, that.detectors) &&
+            Objects.equals(influencers, that.influencers) &&
+            Objects.equals(overlappingBuckets, that.overlappingBuckets) &&
+            Objects.equals(resultFinalizationWindow, that.resultFinalizationWindow) &&
+            Objects.equals(multivariateByFields, that.multivariateByFields);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(
+            bucketSpan, categorizationFieldName, categorizationFilters, categorizationAnalyzerConfig, latency,
+            summaryCountFieldName, detectors, influencers, overlappingBuckets, resultFinalizationWindow,
+            multivariateByFields);
+    }
+
+    public static class Builder {
+
+        private List<Detector> detectors;
+        private TimeValue bucketSpan;
+        private TimeValue latency;
+        private String categorizationFieldName;
+        private List<String> categorizationFilters;
+        private CategorizationAnalyzerConfig categorizationAnalyzerConfig;
+        private String summaryCountFieldName;
+        private List<String> influencers = new ArrayList<>();
+        private Boolean overlappingBuckets;
+        private Long resultFinalizationWindow;
+        private Boolean multivariateByFields;
+
+        public Builder(List<Detector> detectors) {
+            setDetectors(detectors);
+        }
+
+        public Builder(AnalysisConfig analysisConfig) {
+            this.detectors = new ArrayList<>(analysisConfig.detectors);
+            this.bucketSpan = analysisConfig.bucketSpan;
+            this.latency = analysisConfig.latency;
+            this.categorizationFieldName = analysisConfig.categorizationFieldName;
+            this.categorizationFilters = analysisConfig.categorizationFilters == null ? null
+                : new ArrayList<>(analysisConfig.categorizationFilters);
+            this.categorizationAnalyzerConfig = analysisConfig.categorizationAnalyzerConfig;
+            this.summaryCountFieldName = analysisConfig.summaryCountFieldName;
+            this.influencers = new ArrayList<>(analysisConfig.influencers);
+            this.overlappingBuckets = analysisConfig.overlappingBuckets;
+            this.resultFinalizationWindow = analysisConfig.resultFinalizationWindow;
+            this.multivariateByFields = analysisConfig.multivariateByFields;
+        }
+
+        public void setDetectors(List<Detector> detectors) {
+            Objects.requireNonNull(detectors,  "[" + DETECTORS.getPreferredName() + "] must not be null");
+            // We always assign sequential IDs to the detectors that are correct for this analysis config
+            int detectorIndex = 0;
+            List<Detector> sequentialIndexDetectors = new ArrayList<>(detectors.size());
+            for (Detector origDetector : detectors) {
+                Detector.Builder builder = new Detector.Builder(origDetector);
+                builder.setDetectorIndex(detectorIndex++);
+                sequentialIndexDetectors.add(builder.build());
+            }
+            this.detectors = sequentialIndexDetectors;
+        }
+
+        public void setDetector(int detectorIndex, Detector detector) {
+            detectors.set(detectorIndex, detector);
+        }
+
+        public void setBucketSpan(TimeValue bucketSpan) {
+            this.bucketSpan = bucketSpan;
+        }
+
+        public void setLatency(TimeValue latency) {
+            this.latency = latency;
+        }
+
+        public void setCategorizationFieldName(String categorizationFieldName) {
+            this.categorizationFieldName = categorizationFieldName;
+        }
+
+        public void setCategorizationFilters(List<String> categorizationFilters) {
+            this.categorizationFilters = categorizationFilters;
+        }
+
+        public void setCategorizationAnalyzerConfig(CategorizationAnalyzerConfig categorizationAnalyzerConfig) {
+            this.categorizationAnalyzerConfig = categorizationAnalyzerConfig;
+        }
+
+        public void setSummaryCountFieldName(String summaryCountFieldName) {
+            this.summaryCountFieldName = summaryCountFieldName;
+        }
+
+        public void setInfluencers(List<String> influencers) {
+            this.influencers = Objects.requireNonNull(influencers, INFLUENCERS.getPreferredName());
+        }
+
+        public void setOverlappingBuckets(Boolean overlappingBuckets) {
+            this.overlappingBuckets = overlappingBuckets;
+        }
+
+        public void setResultFinalizationWindow(Long resultFinalizationWindow) {
+            this.resultFinalizationWindow = resultFinalizationWindow;
+        }
+
+        public void setMultivariateByFields(Boolean multivariateByFields) {
+            this.multivariateByFields = multivariateByFields;
+        }
+
+        public AnalysisConfig build() {
+
+            return new AnalysisConfig(bucketSpan, categorizationFieldName, categorizationFilters, categorizationAnalyzerConfig,
+                latency, summaryCountFieldName, detectors, influencers, overlappingBuckets,
+                resultFinalizationWindow, multivariateByFields);
+        }
+    }
+}

x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/Job.java

@@ -19,7 +19,572 @@
 package org.elasticsearch.protocol.xpack.ml.job.config;
 
 import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.protocol.xpack.ml.job.util.TimeUtil;
 
-public class Job {
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * This class represents a configured and created Job. The creation time is set
+ * to the time the object was constructed and the finished time and last
+ * data time fields are {@code null} until the job has seen some data or it is
+ * finished respectively.
+ */
+public class Job implements ToXContentObject {
+
+    public static final String ANOMALY_DETECTOR_JOB_TYPE = "anomaly_detector";
+
+    /*
+     * Field names used in serialization
+     */
     public static final ParseField ID = new ParseField("job_id");
+    public static final ParseField JOB_TYPE = new ParseField("job_type");
+    public static final ParseField GROUPS = new ParseField("groups");
+    public static final ParseField ANALYSIS_CONFIG = AnalysisConfig.ANALYSIS_CONFIG;
+    public static final ParseField ANALYSIS_LIMITS = new ParseField("analysis_limits");
+    public static final ParseField CREATE_TIME = new ParseField("create_time");
+    public static final ParseField CUSTOM_SETTINGS = new ParseField("custom_settings");
+    public static final ParseField DATA_DESCRIPTION = new ParseField("data_description");
+    public static final ParseField DESCRIPTION = new ParseField("description");
+    public static final ParseField FINISHED_TIME = new ParseField("finished_time");
+    public static final ParseField LAST_DATA_TIME = new ParseField("last_data_time");
+    public static final ParseField ESTABLISHED_MODEL_MEMORY = new ParseField("established_model_memory");
+    public static final ParseField MODEL_PLOT_CONFIG = new ParseField("model_plot_config");
+    public static final ParseField RENORMALIZATION_WINDOW_DAYS = new ParseField("renormalization_window_days");
+    public static final ParseField BACKGROUND_PERSIST_INTERVAL = new ParseField("background_persist_interval");
+    public static final ParseField MODEL_SNAPSHOT_RETENTION_DAYS = new ParseField("model_snapshot_retention_days");
+    public static final ParseField RESULTS_RETENTION_DAYS = new ParseField("results_retention_days");
+    public static final ParseField MODEL_SNAPSHOT_ID = new ParseField("model_snapshot_id");
+    public static final ParseField RESULTS_INDEX_NAME = new ParseField("results_index_name");
+    public static final ParseField DELETED = new ParseField("deleted");
+
+    public static final ObjectParser<Builder, Void> PARSER = new ObjectParser<>("job_details", true, Builder::new);
+
+    static {
+        PARSER.declareString(Builder::setId, ID);
+        PARSER.declareString(Builder::setJobType, JOB_TYPE);
+        PARSER.declareStringArray(Builder::setGroups, GROUPS);
+        PARSER.declareStringOrNull(Builder::setDescription, DESCRIPTION);
+        PARSER.declareField(Builder::setCreateTime,
+            (p) -> TimeUtil.parseTimeField(p, CREATE_TIME.getPreferredName()),
+            CREATE_TIME,
+            ValueType.VALUE);
+        PARSER.declareField(Builder::setFinishedTime,
+            (p) -> TimeUtil.parseTimeField(p, FINISHED_TIME.getPreferredName()),
+            FINISHED_TIME,
+            ValueType.VALUE);
+        PARSER.declareField(Builder::setLastDataTime,
+            (p) -> TimeUtil.parseTimeField(p, LAST_DATA_TIME.getPreferredName()),
+            LAST_DATA_TIME,
+            ValueType.VALUE);
+        PARSER.declareLong(Builder::setEstablishedModelMemory, ESTABLISHED_MODEL_MEMORY);
+        PARSER.declareObject(Builder::setAnalysisConfig, AnalysisConfig.PARSER, ANALYSIS_CONFIG);
+        PARSER.declareObject(Builder::setAnalysisLimits, AnalysisLimits.PARSER, ANALYSIS_LIMITS);
+        PARSER.declareObject(Builder::setDataDescription, DataDescription.PARSER, DATA_DESCRIPTION);
+        PARSER.declareObject(Builder::setModelPlotConfig, ModelPlotConfig.PARSER, MODEL_PLOT_CONFIG);
+        PARSER.declareLong(Builder::setRenormalizationWindowDays, RENORMALIZATION_WINDOW_DAYS);
+        PARSER.declareString((builder, val) -> builder.setBackgroundPersistInterval(
+            TimeValue.parseTimeValue(val, BACKGROUND_PERSIST_INTERVAL.getPreferredName())), BACKGROUND_PERSIST_INTERVAL);
+        PARSER.declareLong(Builder::setResultsRetentionDays, RESULTS_RETENTION_DAYS);
+        PARSER.declareLong(Builder::setModelSnapshotRetentionDays, MODEL_SNAPSHOT_RETENTION_DAYS);
+        PARSER.declareField(Builder::setCustomSettings, (p, c) -> p.map(), CUSTOM_SETTINGS, ValueType.OBJECT);
+        PARSER.declareStringOrNull(Builder::setModelSnapshotId, MODEL_SNAPSHOT_ID);
+        PARSER.declareString(Builder::setResultsIndexName, RESULTS_INDEX_NAME);
+        PARSER.declareBoolean(Builder::setDeleted, DELETED);
+    }
+
+    private final String jobId;
+    private final String jobType;
+
+    private final List<String> groups;
+    private final String description;
+    private final Date createTime;
+    private final Date finishedTime;
+    private final Date lastDataTime;
+    private final Long establishedModelMemory;
+    private final AnalysisConfig analysisConfig;
+    private final AnalysisLimits analysisLimits;
+    private final DataDescription dataDescription;
+    private final ModelPlotConfig modelPlotConfig;
+    private final Long renormalizationWindowDays;
+    private final TimeValue backgroundPersistInterval;
+    private final Long modelSnapshotRetentionDays;
+    private final Long resultsRetentionDays;
+    private final Map<String, Object> customSettings;
+    private final String modelSnapshotId;
+    private final String resultsIndexName;
+    private final boolean deleted;
+
+    private Job(String jobId, String jobType, List<String> groups, String description, Date createTime,
+                Date finishedTime, Date lastDataTime, Long establishedModelMemory,
+                AnalysisConfig analysisConfig, AnalysisLimits analysisLimits, DataDescription dataDescription,
+                ModelPlotConfig modelPlotConfig, Long renormalizationWindowDays, TimeValue backgroundPersistInterval,
+                Long modelSnapshotRetentionDays, Long resultsRetentionDays, Map<String, Object> customSettings,
+                String modelSnapshotId, String resultsIndexName, boolean deleted) {
+
+        this.jobId = jobId;
+        this.jobType = jobType;
+        this.groups = Collections.unmodifiableList(groups);
+        this.description = description;
+        this.createTime = createTime;
+        this.finishedTime = finishedTime;
+        this.lastDataTime = lastDataTime;
+        this.establishedModelMemory = establishedModelMemory;
+        this.analysisConfig = analysisConfig;
+        this.analysisLimits = analysisLimits;
+        this.dataDescription = dataDescription;
+        this.modelPlotConfig = modelPlotConfig;
+        this.renormalizationWindowDays = renormalizationWindowDays;
+        this.backgroundPersistInterval = backgroundPersistInterval;
+        this.modelSnapshotRetentionDays = modelSnapshotRetentionDays;
+        this.resultsRetentionDays = resultsRetentionDays;
+        this.customSettings = customSettings == null ? null : Collections.unmodifiableMap(customSettings);
+        this.modelSnapshotId = modelSnapshotId;
+        this.resultsIndexName = resultsIndexName;
+        this.deleted = deleted;
+    }
+
+    /**
+     * Return the Job Id.
+     *
+     * @return The job Id string
+     */
+    public String getId() {
+        return jobId;
+    }
+
+    public String getJobType() {
+        return jobType;
+    }
+
+    public List<String> getGroups() {
+        return groups;
+    }
+
+    /**
+     * Private version of getResultsIndexName so that a job can be built from another
+     * job and pass index name validation
+     *
+     * @return The job's index name, minus prefix
+     */
+    private String getResultsIndexNameNoPrefix() {
+        return resultsIndexName;
+    }
+
+    /**
+     * The job description
+     *
+     * @return job description
+     */
+    public String getDescription() {
+        return description;
+    }
+
+    /**
+     * The Job creation time. This name is preferred when serialising to the
+     * REST API.
+     *
+     * @return The date the job was created
+     */
+    public Date getCreateTime() {
+        return createTime;
+    }
+
+    /**
+     * The time the job was finished or <code>null</code> if not finished.
+     *
+     * @return The date the job was last retired or <code>null</code>
+     */
+    public Date getFinishedTime() {
+        return finishedTime;
+    }
+
+    /**
+     * The last time data was uploaded to the job or <code>null</code> if no
+     * data has been seen.
+     *
+     * @return The date at which the last data was processed
+     */
+    public Date getLastDataTime() {
+        return lastDataTime;
+    }
+
+    /**
+     * The established model memory of the job, or <code>null</code> if model
+     * memory has not reached equilibrium yet.
+     *
+     * @return The established model memory of the job
+     */
+    public Long getEstablishedModelMemory() {
+        return establishedModelMemory;
+    }
+
+    /**
+     * The analysis configuration object
+     *
+     * @return The AnalysisConfig
+     */
+    public AnalysisConfig getAnalysisConfig() {
+        return analysisConfig;
+    }
+
+    /**
+     * The analysis options object
+     *
+     * @return The AnalysisLimits
+     */
+    public AnalysisLimits getAnalysisLimits() {
+        return analysisLimits;
+    }
+
+    public ModelPlotConfig getModelPlotConfig() {
+        return modelPlotConfig;
+    }
+
+    /**
+     * If not set the input data is assumed to be csv with a '_time' field in
+     * epoch format.
+     *
+     * @return A DataDescription or <code>null</code>
+     * @see DataDescription
+     */
+    public DataDescription getDataDescription() {
+        return dataDescription;
+    }
+
+    /**
+     * The duration of the renormalization window in days
+     *
+     * @return renormalization window in days
+     */
+    public Long getRenormalizationWindowDays() {
+        return renormalizationWindowDays;
+    }
+
+    /**
+     * The background persistence interval
+     *
+     * @return background persistence interval
+     */
+    public TimeValue getBackgroundPersistInterval() {
+        return backgroundPersistInterval;
+    }
+
+    public Long getModelSnapshotRetentionDays() {
+        return modelSnapshotRetentionDays;
+    }
+
+    public Long getResultsRetentionDays() {
+        return resultsRetentionDays;
+    }
+
+    public Map<String, Object> getCustomSettings() {
+        return customSettings;
+    }
+
+    public String getModelSnapshotId() {
+        return modelSnapshotId;
+    }
+
+    public boolean isDeleted() {
+        return deleted;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        final String humanReadableSuffix = "_string";
+
+        builder.field(ID.getPreferredName(), jobId);
+        builder.field(JOB_TYPE.getPreferredName(), jobType);
+
+        if (groups.isEmpty() == false) {
+            builder.field(GROUPS.getPreferredName(), groups);
+        }
+        if (description != null) {
+            builder.field(DESCRIPTION.getPreferredName(), description);
+        }
+        builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + humanReadableSuffix, createTime.getTime());
+        if (finishedTime != null) {
+            builder.timeField(FINISHED_TIME.getPreferredName(), FINISHED_TIME.getPreferredName() + humanReadableSuffix,
+                finishedTime.getTime());
+        }
+        if (lastDataTime != null) {
+            builder.timeField(LAST_DATA_TIME.getPreferredName(), LAST_DATA_TIME.getPreferredName() + humanReadableSuffix,
+                lastDataTime.getTime());
+        }
+        if (establishedModelMemory != null) {
+            builder.field(ESTABLISHED_MODEL_MEMORY.getPreferredName(), establishedModelMemory);
+        }
+        builder.field(ANALYSIS_CONFIG.getPreferredName(), analysisConfig, params);
+        if (analysisLimits != null) {
+            builder.field(ANALYSIS_LIMITS.getPreferredName(), analysisLimits, params);
+        }
+        if (dataDescription != null) {
+            builder.field(DATA_DESCRIPTION.getPreferredName(), dataDescription, params);
+        }
+        if (modelPlotConfig != null) {
+            builder.field(MODEL_PLOT_CONFIG.getPreferredName(), modelPlotConfig, params);
+        }
+        if (renormalizationWindowDays != null) {
+            builder.field(RENORMALIZATION_WINDOW_DAYS.getPreferredName(), renormalizationWindowDays);
+        }
+        if (backgroundPersistInterval != null) {
+            builder.field(BACKGROUND_PERSIST_INTERVAL.getPreferredName(), backgroundPersistInterval.getStringRep());
+        }
+        if (modelSnapshotRetentionDays != null) {
+            builder.field(MODEL_SNAPSHOT_RETENTION_DAYS.getPreferredName(), modelSnapshotRetentionDays);
+        }
+        if (resultsRetentionDays != null) {
+            builder.field(RESULTS_RETENTION_DAYS.getPreferredName(), resultsRetentionDays);
+        }
+        if (customSettings != null) {
+            builder.field(CUSTOM_SETTINGS.getPreferredName(), customSettings);
+        }
+        if (modelSnapshotId != null) {
+            builder.field(MODEL_SNAPSHOT_ID.getPreferredName(), modelSnapshotId);
+        }
+        if (resultsIndexName != null) {
+            builder.field(RESULTS_INDEX_NAME.getPreferredName(), resultsIndexName);
+        }
+        if (params.paramAsBoolean("all", false)) {
+            builder.field(DELETED.getPreferredName(), deleted);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+
+        if (other == null || getClass() != other.getClass()) {
+            return false;
+        }
+
+        Job that = (Job) other;
+        return Objects.equals(this.jobId, that.jobId)
+            && Objects.equals(this.jobType, that.jobType)
+            && Objects.equals(this.groups, that.groups)
+            && Objects.equals(this.description, that.description)
+            && Objects.equals(this.createTime, that.createTime)
+            && Objects.equals(this.finishedTime, that.finishedTime)
+            && Objects.equals(this.lastDataTime, that.lastDataTime)
+            && Objects.equals(this.establishedModelMemory, that.establishedModelMemory)
+            && Objects.equals(this.analysisConfig, that.analysisConfig)
+            && Objects.equals(this.analysisLimits, that.analysisLimits)
+            && Objects.equals(this.dataDescription, that.dataDescription)
+            && Objects.equals(this.modelPlotConfig, that.modelPlotConfig)
+            && Objects.equals(this.renormalizationWindowDays, that.renormalizationWindowDays)
+            && Objects.equals(this.backgroundPersistInterval, that.backgroundPersistInterval)
+            && Objects.equals(this.modelSnapshotRetentionDays, that.modelSnapshotRetentionDays)
+            && Objects.equals(this.resultsRetentionDays, that.resultsRetentionDays)
+            && Objects.equals(this.customSettings, that.customSettings)
+            && Objects.equals(this.modelSnapshotId, that.modelSnapshotId)
+            && Objects.equals(this.resultsIndexName, that.resultsIndexName)
+            && Objects.equals(this.deleted, that.deleted);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(jobId, jobType, groups, description, createTime, finishedTime, lastDataTime, establishedModelMemory,
+            analysisConfig, analysisLimits, dataDescription, modelPlotConfig, renormalizationWindowDays,
+            backgroundPersistInterval, modelSnapshotRetentionDays, resultsRetentionDays, customSettings,
+            modelSnapshotId, resultsIndexName, deleted);
+    }
+
+    @Override
+    public final String toString() {
+        return Strings.toString(this);
+    }
+
+    public static class Builder {
+
+        private String id;
+        private String jobType = ANOMALY_DETECTOR_JOB_TYPE;
+        private List<String> groups = Collections.emptyList();
+        private String description;
+        private AnalysisConfig analysisConfig;
+        private AnalysisLimits analysisLimits;
+        private DataDescription dataDescription;
+        private Date createTime;
+        private Date finishedTime;
+        private Date lastDataTime;
+        private Long establishedModelMemory;
+        private ModelPlotConfig modelPlotConfig;
+        private Long renormalizationWindowDays;
+        private TimeValue backgroundPersistInterval;
+        private Long modelSnapshotRetentionDays;
+        private Long resultsRetentionDays;
+        private Map<String, Object> customSettings;
+        private String modelSnapshotId;
+        private String resultsIndexName;
+        private boolean deleted;
+
+        public Builder() {
+        }
+
+        public Builder(String id) {
+            this.id = id;
+        }
+
+        public Builder(Job job) {
+            this.id = job.getId();
+            this.jobType = job.getJobType();
+            this.groups = job.getGroups();
+            this.description = job.getDescription();
+            this.analysisConfig = job.getAnalysisConfig();
+            this.analysisLimits = job.getAnalysisLimits();
+            this.dataDescription = job.getDataDescription();
+            this.createTime = job.getCreateTime();
+            this.finishedTime = job.getFinishedTime();
+            this.lastDataTime = job.getLastDataTime();
+            this.establishedModelMemory = job.getEstablishedModelMemory();
+            this.modelPlotConfig = job.getModelPlotConfig();
+            this.renormalizationWindowDays = job.getRenormalizationWindowDays();
+            this.backgroundPersistInterval = job.getBackgroundPersistInterval();
+            this.modelSnapshotRetentionDays = job.getModelSnapshotRetentionDays();
+            this.resultsRetentionDays = job.getResultsRetentionDays();
+            this.customSettings = job.getCustomSettings();
+            this.modelSnapshotId = job.getModelSnapshotId();
+            this.resultsIndexName = job.getResultsIndexNameNoPrefix();
+            this.deleted = job.isDeleted();
+        }
+
+        public Builder setId(String id) {
+            this.id = id;
+            return this;
+        }
+
+        public String getId() {
+            return id;
+        }
+
+        public Builder setJobType(String jobType) {
+            this.jobType = jobType;
+            return this;
+        }
+
+        public Builder setGroups(List<String> groups) {
+            this.groups = groups == null ? Collections.emptyList() : groups;
+            return this;
+        }
+
+        public Builder setCustomSettings(Map<String, Object> customSettings) {
+            this.customSettings = customSettings;
+            return this;
+        }
+
+        public Builder setDescription(String description) {
+            this.description = description;
+            return this;
+        }
+
+        public Builder setAnalysisConfig(AnalysisConfig.Builder configBuilder) {
+            analysisConfig = Objects.requireNonNull(configBuilder, ANALYSIS_CONFIG.getPreferredName()).build();
+            return this;
+        }
+
+        public Builder setAnalysisLimits(AnalysisLimits analysisLimits) {
+            this.analysisLimits = Objects.requireNonNull(analysisLimits, ANALYSIS_LIMITS.getPreferredName());
+            return this;
+        }
+
+        Builder setCreateTime(Date createTime) {
+            this.createTime = createTime;
+            return this;
+        }
+
+        Builder setFinishedTime(Date finishedTime) {
+            this.finishedTime = finishedTime;
+            return this;
+        }
+
+        /**
+         * Set the wall clock time of the last data upload
+         *
+         * @param lastDataTime Wall clock time
+         */
+        public Builder setLastDataTime(Date lastDataTime) {
+            this.lastDataTime = lastDataTime;
+            return this;
+        }
+
+        public Builder setEstablishedModelMemory(Long establishedModelMemory) {
+            this.establishedModelMemory = establishedModelMemory;
+            return this;
+        }
+
+        public Builder setDataDescription(DataDescription.Builder description) {
+            dataDescription = Objects.requireNonNull(description, DATA_DESCRIPTION.getPreferredName()).build();
+            return this;
+        }
+
+        public Builder setModelPlotConfig(ModelPlotConfig modelPlotConfig) {
+            this.modelPlotConfig = modelPlotConfig;
+            return this;
+        }
+
+        public Builder setBackgroundPersistInterval(TimeValue backgroundPersistInterval) {
+            this.backgroundPersistInterval = backgroundPersistInterval;
+            return this;
+        }
+
+        public Builder setRenormalizationWindowDays(Long renormalizationWindowDays) {
+            this.renormalizationWindowDays = renormalizationWindowDays;
+            return this;
+        }
+
+        public Builder setModelSnapshotRetentionDays(Long modelSnapshotRetentionDays) {
+            this.modelSnapshotRetentionDays = modelSnapshotRetentionDays;
+            return this;
+        }
+
+        public Builder setResultsRetentionDays(Long resultsRetentionDays) {
+            this.resultsRetentionDays = resultsRetentionDays;
+            return this;
+        }
+
+        public Builder setModelSnapshotId(String modelSnapshotId) {
+            this.modelSnapshotId = modelSnapshotId;
+            return this;
+        }
+
+        public Builder setResultsIndexName(String resultsIndexName) {
+            this.resultsIndexName = resultsIndexName;
+            return this;
+        }
+
+        public Builder setDeleted(boolean deleted) {
+            this.deleted = deleted;
+            return this;
+        }
+
+        /**
+         * Builds a job.
+         *
+         * @return The job
+         */
+        public Job build() {
+            Objects.requireNonNull(id,  "[" + ID.getPreferredName() + "] must not be null");
+            Objects.requireNonNull(jobType,  "[" + JOB_TYPE.getPreferredName() + "] must not be null");
+            return new Job(
+                id, jobType, groups, description, createTime, finishedTime, lastDataTime, establishedModelMemory,
+                analysisConfig, analysisLimits, dataDescription, modelPlotConfig, renormalizationWindowDays,
+                backgroundPersistInterval, modelSnapshotRetentionDays, resultsRetentionDays, customSettings,
+                modelSnapshotId, resultsIndexName, deleted);
+        }
+    }
 }

x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/process/DataCounts.java

@@ -24,6 +24,7 @@ import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.protocol.xpack.ml.job.config.Job;
+import org.elasticsearch.protocol.xpack.ml.job.util.TimeUtil;
 
 import java.io.IOException;
 import java.util.Date;

x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/process/ModelSizeStats.java

@@ -25,6 +25,7 @@ import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.protocol.xpack.ml.job.config.Job;
 import org.elasticsearch.protocol.xpack.ml.job.results.Result;
+import org.elasticsearch.protocol.xpack.ml.job.util.TimeUtil;
 
 import java.io.IOException;
 import java.util.Date;

x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/process/ModelSnapshot.java

@@ -25,6 +25,7 @@ import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.protocol.xpack.ml.job.config.Job;
+import org.elasticsearch.protocol.xpack.ml.job.util.TimeUtil;
 
 import java.io.IOException;
 import java.util.Date;

x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/process/TimeUtil.java → x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/util/TimeUtil.java

@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.protocol.xpack.ml.job.process;
+package org.elasticsearch.protocol.xpack.ml.job.util;
 
 import org.elasticsearch.common.time.DateFormatters;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -25,7 +25,7 @@ import java.io.IOException;
 import java.time.format.DateTimeFormatter;
 import java.util.Date;
 
-final class TimeUtil {
+public final class TimeUtil {
 
     /**
      * Parse out a Date object given the current parser and field name.
@@ -35,7 +35,7 @@ final class TimeUtil {
      * @return parsed Date object
      * @throws IOException from XContentParser
      */
-    static Date parseTimeField(XContentParser parser, String fieldName) throws IOException {
+    public static Date parseTimeField(XContentParser parser, String fieldName) throws IOException {
         if (parser.currentToken() == XContentParser.Token.VALUE_NUMBER) {
             return new Date(parser.longValue());
         } else if (parser.currentToken() == XContentParser.Token.VALUE_STRING) {

x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/AnalysisConfigTests.java

@@ -0,0 +1,268 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.config;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class AnalysisConfigTests extends AbstractXContentTestCase<AnalysisConfig> {
+
+    public static AnalysisConfig.Builder createRandomized() {
+        boolean isCategorization = randomBoolean();
+        List<Detector> detectors = new ArrayList<>();
+        int numDetectors = randomIntBetween(1, 10);
+        for (int i = 0; i < numDetectors; i++) {
+            Detector.Builder builder = new Detector.Builder("count", null);
+            builder.setPartitionFieldName(isCategorization ? "mlcategory" : "part");
+            detectors.add(builder.build());
+        }
+        AnalysisConfig.Builder builder = new AnalysisConfig.Builder(detectors);
+        if (randomBoolean()) {
+            TimeValue bucketSpan = TimeValue.timeValueSeconds(randomIntBetween(1, 1_000_000));
+            builder.setBucketSpan(bucketSpan);
+        }
+        if (isCategorization) {
+            builder.setCategorizationFieldName(randomAlphaOfLength(10));
+            if (randomBoolean()) {
+                builder.setCategorizationFilters(Arrays.asList(generateRandomStringArray(10, 10, false)));
+            } else {
+                CategorizationAnalyzerConfig.Builder analyzerBuilder = new CategorizationAnalyzerConfig.Builder();
+                if (rarely()) {
+                    analyzerBuilder.setAnalyzer(randomAlphaOfLength(10));
+                } else {
+                    if (randomBoolean()) {
+                        for (String pattern : generateRandomStringArray(3, 40, false)) {
+                            Map<String, Object> charFilter = new HashMap<>();
+                            charFilter.put("type", "pattern_replace");
+                            charFilter.put("pattern", pattern);
+                            analyzerBuilder.addCharFilter(charFilter);
+                        }
+                    }
+
+                    Map<String, Object> tokenizer = new HashMap<>();
+                    tokenizer.put("type", "pattern");
+                    tokenizer.put("pattern", randomAlphaOfLength(10));
+                    analyzerBuilder.setTokenizer(tokenizer);
+
+                    if (randomBoolean()) {
+                        for (String pattern : generateRandomStringArray(4, 40, false)) {
+                            Map<String, Object> tokenFilter = new HashMap<>();
+                            tokenFilter.put("type", "pattern_replace");
+                            tokenFilter.put("pattern", pattern);
+                            analyzerBuilder.addTokenFilter(tokenFilter);
+                        }
+                    }
+                }
+                builder.setCategorizationAnalyzerConfig(analyzerBuilder.build());
+            }
+        }
+        if (randomBoolean()) {
+            builder.setLatency(TimeValue.timeValueSeconds(randomIntBetween(1, 1_000_000)));
+        }
+        if (randomBoolean()) {
+            builder.setMultivariateByFields(randomBoolean());
+        }
+        if (randomBoolean()) {
+            builder.setOverlappingBuckets(randomBoolean());
+        }
+        if (randomBoolean()) {
+            builder.setResultFinalizationWindow(randomNonNegativeLong());
+        }
+
+        builder.setInfluencers(Arrays.asList(generateRandomStringArray(10, 10, false)));
+        return builder;
+    }
+
+    @Override
+    protected AnalysisConfig createTestInstance() {
+        return createRandomized().build();
+    }
+
+    @Override
+    protected AnalysisConfig doParseInstance(XContentParser parser) {
+        return AnalysisConfig.PARSER.apply(parser, null).build();
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+
+    public void testBuilder_WithNullDetectors() {
+        AnalysisConfig.Builder builder = new AnalysisConfig.Builder(new ArrayList<>());
+        NullPointerException ex = expectThrows(NullPointerException.class, () ->  builder.setDetectors(null));
+        assertEquals("[detectors] must not be null", ex.getMessage());
+    }
+
+    public void testEquals_GivenSameReference() {
+        AnalysisConfig config = createRandomized().build();
+        assertTrue(config.equals(config));
+    }
+
+    public void testEquals_GivenDifferentClass() {
+        assertFalse(createRandomized().build().equals("a string"));
+    }
+
+    public void testEquals_GivenNull() {
+        assertFalse(createRandomized().build().equals(null));
+    }
+
+    public void testEquals_GivenEqualConfig() {
+        AnalysisConfig config1 = createValidCategorizationConfig().build();
+        AnalysisConfig config2 = createValidCategorizationConfig().build();
+
+        assertTrue(config1.equals(config2));
+        assertTrue(config2.equals(config1));
+        assertEquals(config1.hashCode(), config2.hashCode());
+    }
+
+    public void testEquals_GivenDifferentBucketSpan() {
+        AnalysisConfig.Builder builder = createConfigBuilder();
+        builder.setBucketSpan(TimeValue.timeValueSeconds(1800));
+        AnalysisConfig config1 = builder.build();
+
+        builder = createConfigBuilder();
+        builder.setBucketSpan(TimeValue.timeValueHours(1));
+        AnalysisConfig config2 = builder.build();
+
+        assertFalse(config1.equals(config2));
+        assertFalse(config2.equals(config1));
+    }
+
+    public void testEquals_GivenCategorizationField() {
+        AnalysisConfig.Builder builder = createValidCategorizationConfig();
+        builder.setCategorizationFieldName("foo");
+        AnalysisConfig config1 = builder.build();
+
+        builder = createValidCategorizationConfig();
+        builder.setCategorizationFieldName("bar");
+        AnalysisConfig config2 = builder.build();
+
+        assertFalse(config1.equals(config2));
+        assertFalse(config2.equals(config1));
+    }
+
+    public void testEquals_GivenDifferentDetector() {
+        AnalysisConfig config1 = createConfigWithDetectors(Collections.singletonList(new Detector.Builder("min", "low_count").build()));
+
+        AnalysisConfig config2 = createConfigWithDetectors(Collections.singletonList(new Detector.Builder("min", "high_count").build()));
+
+        assertFalse(config1.equals(config2));
+        assertFalse(config2.equals(config1));
+    }
+
+    public void testEquals_GivenDifferentInfluencers() {
+        AnalysisConfig.Builder builder = createConfigBuilder();
+        builder.setInfluencers(Collections.singletonList("foo"));
+        AnalysisConfig config1 = builder.build();
+
+        builder = createConfigBuilder();
+        builder.setInfluencers(Collections.singletonList("bar"));
+        AnalysisConfig config2 = builder.build();
+
+        assertFalse(config1.equals(config2));
+        assertFalse(config2.equals(config1));
+    }
+
+    public void testEquals_GivenDifferentLatency() {
+        AnalysisConfig.Builder builder = createConfigBuilder();
+        builder.setLatency(TimeValue.timeValueSeconds(1800));
+        AnalysisConfig config1 = builder.build();
+
+        builder = createConfigBuilder();
+        builder.setLatency(TimeValue.timeValueSeconds(1801));
+        AnalysisConfig config2 = builder.build();
+
+        assertFalse(config1.equals(config2));
+        assertFalse(config2.equals(config1));
+    }
+
+    public void testEquals_GivenSummaryCountField() {
+        AnalysisConfig.Builder builder = createConfigBuilder();
+        builder.setSummaryCountFieldName("foo");
+        AnalysisConfig config1 = builder.build();
+
+        builder = createConfigBuilder();
+        builder.setSummaryCountFieldName("bar");
+        AnalysisConfig config2 = builder.build();
+
+        assertFalse(config1.equals(config2));
+        assertFalse(config2.equals(config1));
+    }
+
+    public void testEquals_GivenMultivariateByField() {
+        AnalysisConfig.Builder builder = createConfigBuilder();
+        builder.setMultivariateByFields(true);
+        AnalysisConfig config1 = builder.build();
+
+        builder = createConfigBuilder();
+        builder.setMultivariateByFields(false);
+        AnalysisConfig config2 = builder.build();
+
+        assertFalse(config1.equals(config2));
+        assertFalse(config2.equals(config1));
+    }
+
+    public void testEquals_GivenDifferentCategorizationFilters() {
+        AnalysisConfig.Builder configBuilder1 = createValidCategorizationConfig();
+        AnalysisConfig.Builder configBuilder2 = createValidCategorizationConfig();
+        configBuilder1.setCategorizationFilters(Arrays.asList("foo", "bar"));
+        configBuilder2.setCategorizationFilters(Arrays.asList("foo", "foobar"));
+        AnalysisConfig config1 = configBuilder1.build();
+        AnalysisConfig config2 = configBuilder2.build();
+
+        assertFalse(config1.equals(config2));
+        assertFalse(config2.equals(config1));
+    }
+
+    private static AnalysisConfig createConfigWithDetectors(List<Detector> detectors) {
+        return new AnalysisConfig.Builder(detectors).build();
+    }
+
+    private static AnalysisConfig.Builder createConfigBuilder() {
+        return new AnalysisConfig.Builder(Collections.singletonList(new Detector.Builder("min", "count").build()));
+    }
+
+    private static AnalysisConfig.Builder createValidCategorizationConfig() {
+        Detector.Builder detector = new Detector.Builder("count", null);
+        detector.setByFieldName("mlcategory");
+        AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(Collections.singletonList(detector.build()));
+        analysisConfig.setBucketSpan(TimeValue.timeValueHours(1));
+        analysisConfig.setLatency(TimeValue.ZERO);
+        analysisConfig.setCategorizationFieldName("msg");
+        return analysisConfig;
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        return new NamedXContentRegistry(searchModule.getNamedXContents());
+    }
+}

x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/config/JobTests.java

@@ -0,0 +1,276 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.protocol.xpack.ml.job.config;
+
+import com.carrotsearch.randomizedtesting.generators.CodepointSetGenerator;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.DeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class JobTests extends AbstractXContentTestCase<Job> {
+
+    private static final String FUTURE_JOB = "{\n" +
+        "    \"job_id\": \"farequote\",\n" +
+        "    \"create_time\": 1234567890000,\n" +
+        "    \"tomorrows_technology_today\": \"wow\",\n" +
+        "    \"analysis_config\": {\n" +
+        "        \"bucket_span\": \"1h\",\n" +
+        "        \"something_new\": \"gasp\",\n" +
+        "        \"detectors\": [{\"function\": \"metric\", \"field_name\": \"responsetime\", \"by_field_name\": \"airline\"}]\n" +
+        "    },\n" +
+        "    \"data_description\": {\n" +
+        "        \"time_field\": \"time\",\n" +
+        "        \"the_future\": 123\n" +
+        "    }\n" +
+        "}";
+
+    @Override
+    protected Job createTestInstance() {
+        return createRandomizedJob();
+    }
+
+    @Override
+    protected Job doParseInstance(XContentParser parser) {
+        return Job.PARSER.apply(parser, null).build();
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+
+    public void testFutureMetadataParse() throws IOException {
+        XContentParser parser = XContentFactory.xContent(XContentType.JSON)
+            .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, FUTURE_JOB);
+        // The parser should tolerate unknown fields
+        assertNotNull(Job.PARSER.apply(parser, null).build());
+    }
+
+    public void testEquals_GivenDifferentClass() {
+        Job job = buildJobBuilder("foo").build();
+        assertFalse(job.equals("a string"));
+    }
+
+    public void testEquals_GivenDifferentIds() {
+        Date createTime = new Date();
+        Job.Builder builder = buildJobBuilder("foo");
+        builder.setCreateTime(createTime);
+        Job job1 = builder.build();
+        builder.setId("bar");
+        Job job2 = builder.build();
+        assertFalse(job1.equals(job2));
+    }
+
+    public void testEquals_GivenDifferentRenormalizationWindowDays() {
+        Date date = new Date();
+        Job.Builder jobDetails1 = new Job.Builder("foo");
+        jobDetails1.setDataDescription(new DataDescription.Builder());
+        jobDetails1.setAnalysisConfig(createAnalysisConfig());
+        jobDetails1.setRenormalizationWindowDays(3L);
+        jobDetails1.setCreateTime(date);
+        Job.Builder jobDetails2 = new Job.Builder("foo");
+        jobDetails2.setDataDescription(new DataDescription.Builder());
+        jobDetails2.setRenormalizationWindowDays(4L);
+        jobDetails2.setAnalysisConfig(createAnalysisConfig());
+        jobDetails2.setCreateTime(date);
+        assertFalse(jobDetails1.build().equals(jobDetails2.build()));
+    }
+
+    public void testEquals_GivenDifferentBackgroundPersistInterval() {
+        Date date = new Date();
+        Job.Builder jobDetails1 = new Job.Builder("foo");
+        jobDetails1.setDataDescription(new DataDescription.Builder());
+        jobDetails1.setAnalysisConfig(createAnalysisConfig());
+        jobDetails1.setBackgroundPersistInterval(TimeValue.timeValueSeconds(10000L));
+        jobDetails1.setCreateTime(date);
+        Job.Builder jobDetails2 = new Job.Builder("foo");
+        jobDetails2.setDataDescription(new DataDescription.Builder());
+        jobDetails2.setBackgroundPersistInterval(TimeValue.timeValueSeconds(8000L));
+        jobDetails2.setAnalysisConfig(createAnalysisConfig());
+        jobDetails2.setCreateTime(date);
+        assertFalse(jobDetails1.build().equals(jobDetails2.build()));
+    }
+
+    public void testEquals_GivenDifferentModelSnapshotRetentionDays() {
+        Date date = new Date();
+        Job.Builder jobDetails1 = new Job.Builder("foo");
+        jobDetails1.setDataDescription(new DataDescription.Builder());
+        jobDetails1.setAnalysisConfig(createAnalysisConfig());
+        jobDetails1.setModelSnapshotRetentionDays(10L);
+        jobDetails1.setCreateTime(date);
+        Job.Builder jobDetails2 = new Job.Builder("foo");
+        jobDetails2.setDataDescription(new DataDescription.Builder());
+        jobDetails2.setModelSnapshotRetentionDays(8L);
+        jobDetails2.setAnalysisConfig(createAnalysisConfig());
+        jobDetails2.setCreateTime(date);
+        assertFalse(jobDetails1.build().equals(jobDetails2.build()));
+    }
+
+    public void testEquals_GivenDifferentResultsRetentionDays() {
+        Date date = new Date();
+        Job.Builder jobDetails1 = new Job.Builder("foo");
+        jobDetails1.setDataDescription(new DataDescription.Builder());
+        jobDetails1.setAnalysisConfig(createAnalysisConfig());
+        jobDetails1.setCreateTime(date);
+        jobDetails1.setResultsRetentionDays(30L);
+        Job.Builder jobDetails2 = new Job.Builder("foo");
+        jobDetails2.setDataDescription(new DataDescription.Builder());
+        jobDetails2.setResultsRetentionDays(4L);
+        jobDetails2.setAnalysisConfig(createAnalysisConfig());
+        jobDetails2.setCreateTime(date);
+        assertFalse(jobDetails1.build().equals(jobDetails2.build()));
+    }
+
+    public void testEquals_GivenDifferentCustomSettings() {
+        Job.Builder jobDetails1 = buildJobBuilder("foo");
+        Map<String, Object> customSettings1 = new HashMap<>();
+        customSettings1.put("key1", "value1");
+        jobDetails1.setCustomSettings(customSettings1);
+        Job.Builder jobDetails2 = buildJobBuilder("foo");
+        Map<String, Object> customSettings2 = new HashMap<>();
+        customSettings2.put("key2", "value2");
+        jobDetails2.setCustomSettings(customSettings2);
+        assertFalse(jobDetails1.build().equals(jobDetails2.build()));
+    }
+
+    public void testCopyConstructor() {
+        for (int i = 0; i < NUMBER_OF_TEST_RUNS; i++) {
+            Job job = createTestInstance();
+            Job copy = new Job.Builder(job).build();
+            assertEquals(job, copy);
+        }
+    }
+
+    public void testBuilder_WithNullID() {
+        Job.Builder builder = new Job.Builder("anything").setId(null);
+        NullPointerException ex = expectThrows(NullPointerException.class, builder::build);
+        assertEquals("[job_id] must not be null", ex.getMessage());
+    }
+
+    public void testBuilder_WithNullJobType() {
+        Job.Builder builder = new Job.Builder("anything").setJobType(null);
+        NullPointerException ex = expectThrows(NullPointerException.class, builder::build);
+        assertEquals("[job_type] must not be null", ex.getMessage());
+    }
+
+    public static Job.Builder buildJobBuilder(String id, Date date) {
+        Job.Builder builder = new Job.Builder(id);
+        builder.setCreateTime(date);
+        AnalysisConfig.Builder ac = createAnalysisConfig();
+        DataDescription.Builder dc = new DataDescription.Builder();
+        builder.setAnalysisConfig(ac);
+        builder.setDataDescription(dc);
+        return builder;
+    }
+
+    public static Job.Builder buildJobBuilder(String id) {
+        return buildJobBuilder(id, new Date());
+    }
+
+    public static String randomValidJobId() {
+        CodepointSetGenerator generator = new CodepointSetGenerator("abcdefghijklmnopqrstuvwxyz".toCharArray());
+        return generator.ofCodePointsLength(random(), 10, 10);
+    }
+
+    public static AnalysisConfig.Builder createAnalysisConfig() {
+        Detector.Builder d1 = new Detector.Builder("info_content", "domain");
+        d1.setOverFieldName("client");
+        Detector.Builder d2 = new Detector.Builder("min", "field");
+        return new AnalysisConfig.Builder(Arrays.asList(d1.build(), d2.build()));
+    }
+
+    public static Job createRandomizedJob() {
+        String jobId = randomValidJobId();
+        Job.Builder builder = new Job.Builder(jobId);
+        if (randomBoolean()) {
+            builder.setDescription(randomAlphaOfLength(10));
+        }
+        if (randomBoolean()) {
+            int groupsNum = randomIntBetween(0, 10);
+            List<String> groups = new ArrayList<>(groupsNum);
+            for (int i = 0; i < groupsNum; i++) {
+                groups.add(randomValidJobId());
+            }
+            builder.setGroups(groups);
+        }
+        builder.setCreateTime(new Date(randomNonNegativeLong()));
+        if (randomBoolean()) {
+            builder.setFinishedTime(new Date(randomNonNegativeLong()));
+        }
+        if (randomBoolean()) {
+            builder.setLastDataTime(new Date(randomNonNegativeLong()));
+        }
+        if (randomBoolean()) {
+            builder.setEstablishedModelMemory(randomNonNegativeLong());
+        }
+        builder.setAnalysisConfig(AnalysisConfigTests.createRandomized());
+        builder.setAnalysisLimits(AnalysisLimitsTests.createRandomized());
+
+        DataDescription.Builder dataDescription = new DataDescription.Builder();
+        dataDescription.setFormat(randomFrom(DataDescription.DataFormat.values()));
+        builder.setDataDescription(dataDescription);
+
+        if (randomBoolean()) {
+            builder.setModelPlotConfig(new ModelPlotConfig(randomBoolean(), randomAlphaOfLength(10)));
+        }
+        if (randomBoolean()) {
+            builder.setRenormalizationWindowDays(randomNonNegativeLong());
+        }
+        if (randomBoolean()) {
+            builder.setBackgroundPersistInterval(TimeValue.timeValueHours(randomIntBetween(1, 24)));
+        }
+        if (randomBoolean()) {
+            builder.setModelSnapshotRetentionDays(randomNonNegativeLong());
+        }
+        if (randomBoolean()) {
+            builder.setResultsRetentionDays(randomNonNegativeLong());
+        }
+        if (randomBoolean()) {
+            builder.setCustomSettings(Collections.singletonMap(randomAlphaOfLength(10), randomAlphaOfLength(10)));
+        }
+        if (randomBoolean()) {
+            builder.setModelSnapshotId(randomAlphaOfLength(10));
+        }
+        if (randomBoolean()) {
+            builder.setResultsIndexName(randomValidJobId());
+        }
+        return builder.build();
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        return new NamedXContentRegistry(searchModule.getNamedXContents());
+    }
+}