[ML] Set default value of 30 days for model prune window (#81377)

For new jobs, when the analysis config field model_prune_window is not set, use a default value of 30 days or 20 times the bucket span, whichever is greater.

Co-authored-by: David Roberts <dave.roberts@elastic.co>
Co-authored-by: Lisa Cawley <lcawley@elastic.co>

docs/reference/ml/anomaly-detection/apis/put-job.asciidoc

@@ -436,7 +436,8 @@ When the job is created, you receive the following results:
         "detector_index" : 0
       }
     ],
-    "influencers" : [ ]
+    "influencers" : [ ],
+    "model_prune_window": "30d"
   },
   "analysis_limits" : {
     "model_memory_limit" : "1024mb",

docs/reference/ml/ml-shared.asciidoc

@@ -1380,7 +1380,8 @@ Affects the pruning of models that have not been updated for the given time
 duration. The value must be set to a multiple of the `bucket_span`. If set too
 low, important information may be removed from the model. Typically, set to
 `30d` or longer. If not set, model pruning only occurs if the model memory
-status reaches the soft limit or the hard limit.
+status reaches the soft limit or the hard limit. For jobs created in 8.1 and
+later, the default value is the greater of `30d` or 20 times `bucket_span`.
 end::model-prune-window[]
 
 tag::model-snapshot-id[]

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfig.java

@@ -76,6 +76,8 @@ public class AnalysisConfig implements ToXContentObject, Writeable {
     // The minimum number of buckets considered acceptable for the model_prune_window field
     public static final long MINIMUM_MODEL_PRUNE_WINDOW_BUCKETS = 2;
 
+    public static final TimeValue DEFAULT_MODEL_PRUNE_WINDOW = TimeValue.timeValueDays(30);
+
     @SuppressWarnings("unchecked")
     private static ConstructingObjectParser<AnalysisConfig.Builder, Void> createParser(boolean ignoreUnknownFields) {
         ConstructingObjectParser<AnalysisConfig.Builder, Void> parser = new ConstructingObjectParser<>(

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java

@@ -1207,6 +1207,30 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
                 && modelSnapshotRetentionDays > DEFAULT_DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS) {
                 dailyModelSnapshotRetentionAfterDays = DEFAULT_DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS;
             }
+            if (analysisConfig.getModelPruneWindow() == null) {
+                long modelPruneWindowSeconds = analysisConfig.getBucketSpan().seconds() / 2 + AnalysisConfig.DEFAULT_MODEL_PRUNE_WINDOW
+                    .seconds();
+                modelPruneWindowSeconds -= (modelPruneWindowSeconds % analysisConfig.getBucketSpan().seconds());
+                if (modelPruneWindowSeconds < AnalysisConfig.DEFAULT_MODEL_PRUNE_WINDOW.seconds()) {
+                    modelPruneWindowSeconds += analysisConfig.getBucketSpan().seconds();
+                }
+                modelPruneWindowSeconds = Math.max(20 * analysisConfig.getBucketSpan().seconds(), modelPruneWindowSeconds);
+
+                AnalysisConfig.Builder analysisConfigBuilder = new AnalysisConfig.Builder(analysisConfig);
+                final long SECONDS_IN_A_DAY = 86400;
+                final long SECONDS_IN_AN_HOUR = 3600;
+                final long SECONDS_IN_A_MINUTE = 60;
+                if (modelPruneWindowSeconds % SECONDS_IN_A_DAY == 0) {
+                    analysisConfigBuilder.setModelPruneWindow(TimeValue.timeValueDays(modelPruneWindowSeconds / SECONDS_IN_A_DAY));
+                } else if (modelPruneWindowSeconds % SECONDS_IN_AN_HOUR == 0) {
+                    analysisConfigBuilder.setModelPruneWindow(TimeValue.timeValueHours(modelPruneWindowSeconds / SECONDS_IN_AN_HOUR));
+                } else if (modelPruneWindowSeconds % SECONDS_IN_A_MINUTE == 0) {
+                    analysisConfigBuilder.setModelPruneWindow(TimeValue.timeValueMinutes(modelPruneWindowSeconds / SECONDS_IN_A_MINUTE));
+                } else {
+                    analysisConfigBuilder.setModelPruneWindow(TimeValue.timeValueSeconds(modelPruneWindowSeconds));
+                }
+                this.setAnalysisConfig(analysisConfigBuilder);
+            }
         }
 
         /**

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfigTests.java

@@ -438,6 +438,15 @@ public class AnalysisConfigTests extends AbstractSerializingTestCase<AnalysisCon
         assertFalse(config2.equals(config1));
     }
 
+    public void testNoDefaultModelPruneWindow() {
+        AnalysisConfig.Builder builder = createConfigBuilder();
+        AnalysisConfig config1 = builder.build();
+
+        // Explicitly check that a default value for model_prune_window
+        // is NOT set when parsing config documents
+        assertEquals(null, config1.getModelPruneWindow());
+    }
+
     public void testEquals_GivenSummaryCountField() {
         AnalysisConfig.Builder builder = createConfigBuilder();
         builder.setSummaryCountFieldName("foo");

x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java

@@ -11,7 +11,6 @@ import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.cluster.AckedClusterStateUpdateTask;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.Metadata;
@@ -36,7 +35,6 @@ import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xpack.core.ml.MachineLearningField;
 import org.elasticsearch.xpack.core.ml.MlConfigIndex;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
-import org.elasticsearch.xpack.core.ml.action.PutJobAction;
 import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig;
 import org.elasticsearch.xpack.core.ml.job.config.CategorizationAnalyzerConfig;
 import org.elasticsearch.xpack.core.ml.job.config.DataDescription;
@@ -56,7 +54,6 @@ import org.elasticsearch.xpack.ml.job.process.autodetect.UpdateParams;
 import org.elasticsearch.xpack.ml.notifications.AnomalyDetectionAuditor;
 import org.junit.Before;
 import org.mockito.ArgumentCaptor;
-import org.mockito.ArgumentMatchers;
 import org.mockito.Mockito;
 
 import java.io.IOException;
@@ -74,10 +71,8 @@ import java.util.concurrent.atomic.AtomicReference;
 import static org.elasticsearch.xpack.core.ml.job.config.JobTests.buildJobBuilder;
 import static org.elasticsearch.xpack.ml.job.task.OpenJobPersistentTasksExecutorTests.addJobTask;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
@@ -145,45 +140,6 @@ public class JobManagerTests extends ESTestCase {
         assertThat(exceptionHolder.get(), instanceOf(ResourceNotFoundException.class));
     }
 
-    @SuppressWarnings("unchecked")
-    public void testPutJob_AddsCreateTime() throws IOException {
-        MockClientBuilder mockClientBuilder = new MockClientBuilder("cluster-test");
-        JobManager jobManager = createJobManager(mockClientBuilder.build());
-
-        PutJobAction.Request putJobRequest = new PutJobAction.Request(createJob());
-
-        doAnswer(invocation -> {
-            ((AckedClusterStateUpdateTask) invocation.getArguments()[1]).onAllNodesAcked(null);
-            return null;
-        }).when(clusterService).submitStateUpdateTask(ArgumentMatchers.eq("put-job-foo"), any(AckedClusterStateUpdateTask.class));
-
-        ArgumentCaptor<Job> requestCaptor = ArgumentCaptor.forClass(Job.class);
-        doAnswer(invocation -> {
-            ActionListener<Boolean> listener = (ActionListener<Boolean>) invocation.getArguments()[2];
-            listener.onResponse(true);
-            return null;
-        }).when(jobResultsProvider).createJobResultIndex(requestCaptor.capture(), any(ClusterState.class), any(ActionListener.class));
-
-        ClusterState clusterState = createClusterState();
-
-        jobManager.putJob(putJobRequest, analysisRegistry, clusterState, new ActionListener<>() {
-            @Override
-            public void onResponse(PutJobAction.Response response) {
-                Job job = requestCaptor.getValue();
-                assertNotNull(job.getCreateTime());
-                Date now = new Date();
-                // job create time should be within the last second
-                assertThat(now.getTime(), greaterThanOrEqualTo(job.getCreateTime().getTime()));
-                assertThat(now.getTime() - 1000, lessThanOrEqualTo(job.getCreateTime().getTime()));
-            }
-
-            @Override
-            public void onFailure(Exception e) {
-                fail(e.toString());
-            }
-        });
-    }
-
     public void testNotifyFilterChangedGivenNoop() {
         MlFilter filter = MlFilter.builder("my_filter").build();
         MockClientBuilder mockClientBuilder = new MockClientBuilder("cluster-test");

x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/jobs_crud.yml

@@ -59,6 +59,8 @@
   - match: { job_id: "job-crud-test-apis" }
   - match: { analysis_limits.model_memory_limit: "1024mb" }
   - match: { analysis_limits.categorization_examples_limit: 4 }
+  - is_true: create_time
+
 
   - do:
       ml.get_jobs:
@@ -110,6 +112,41 @@
   - match: { job_id: "job-model-prune-window" }
   - match: { analysis_config.bucket_span: "15m" }
   - match: { analysis_config.model_prune_window: "14d" }
+  - is_true: create_time
+
+  - do:
+      ml.put_job:
+        job_id: job-default-model-prune-window
+        body: >
+          {
+            "analysis_config" : {
+                "bucket_span": "15m",
+                "detectors" :[{"function":"count"}]
+            },
+            "data_description" : {
+            }
+          }
+  - match: { job_id: "job-default-model-prune-window" }
+  - match: { analysis_config.bucket_span: "15m" }
+  - match: { analysis_config.model_prune_window: "30d" }
+  - is_true: create_time
+
+  - do:
+      ml.put_job:
+        job_id: job-default-model-prune-window-40h
+        body: >
+          {
+            "analysis_config" : {
+                "bucket_span": "40h",
+                "detectors" :[{"function":"count"}]
+            },
+            "data_description" : {
+            }
+          }
+  - match: { job_id: "job-default-model-prune-window-40h" }
+  - match: { analysis_config.bucket_span: "40h" }
+  - match: { analysis_config.model_prune_window: "800h" }
+  - is_true: create_time
 
   - do:
       catch: /model_prune_window \[29m\] must be a multiple of bucket_span \[15m\]/
@@ -141,7 +178,62 @@
             }
           }
 
+# Here we test that in the case of the job being configured with an
+# unconventional bucket span (17m) that the generated model_prune_window
+# is the closest multiple of the bucket span greater than or equal to 30
+# days (43200m)
+  - do:
+      ml.put_job:
+        job_id: job-default-model-prune-window_with_odd_bucket_span
+        body: >
+          {
+            "analysis_config" : {
+                "bucket_span": "17m",
+                "detectors" :[{"function":"count"}]
+            },
+            "data_description" : {
+            }
+          }
+  - match: { job_id: "job-default-model-prune-window_with_odd_bucket_span" }
+  - match: { analysis_config.bucket_span: "17m" }
+  - match: { analysis_config.model_prune_window: "43214m" }
+  - is_true: create_time
 
+# We expect that the default model_prune_window will be set to
+# 20 * bucket_span if this value is greater than 30 daya
+  - do:
+      ml.put_job:
+        job_id: job-default-model-prune-window_with_large_bucket_span
+        body: >
+          {
+            "analysis_config" : {
+                "bucket_span": "14d",
+                "detectors" :[{"function":"count"}]
+            },
+            "data_description" : {
+            }
+          }
+  - match: { job_id: "job-default-model-prune-window_with_large_bucket_span" }
+  - match: { analysis_config.bucket_span: "14d" }
+  - match: { analysis_config.model_prune_window: "280d" }
+  - is_true: create_time
+
+  - do:
+      ml.put_job:
+        job_id: job-default-model-prune-window_with_small_bucket_span
+        body: >
+          {
+            "analysis_config" : {
+                "bucket_span": "1s",
+                "detectors" :[{"function":"count"}]
+            },
+            "data_description" : {
+            }
+          }
+  - match: { job_id: "job-default-model-prune-window_with_small_bucket_span" }
+  - match: { analysis_config.bucket_span: "1s" }
+  - match: { analysis_config.model_prune_window: "30d" }
+  - is_true: create_time
 ---
 "Test put job with model_memory_limit as string and lazy open":
   - skip:
@@ -163,6 +255,7 @@
           }
   - match: { job_id: "job-model-memory-limit-as-string" }
   - match: { analysis_limits.model_memory_limit: "3072000mb" }
+  - is_true: create_time
 
   # The assumption here is that a 3000GB job will not fit on the test
   # node - increase in future if the test ever fails because of this!
@@ -297,6 +390,7 @@
             }
           }
   - match: { job_id: "jobs-crud-id-already-taken" }
+  - is_true: create_time
 
   - do:
       catch: /resource_already_exists_exception/
@@ -376,6 +470,7 @@
   - match: { analysis_config.categorization_analyzer.char_filter.2.pattern: "cat2.*" }
   - match: { analysis_config.bucket_span: "5m" }
   - match: { analysis_config.model_prune_window: "30d" }
+  - is_true: create_time
 
   - do:
       ml.open_job:
@@ -591,6 +686,7 @@
             }
           }
   - match: { job_id: "jobs-crud-datafeed-job" }
+  - is_true: create_time
 
   - do:
       ml.put_datafeed:
@@ -626,6 +722,7 @@
             }
           }
   - match: { job_id: "delete-opened-job" }
+  - is_true: create_time
 
   - do:
       ml.open_job:
@@ -661,6 +758,7 @@
             }
           }
   - match: { job_id: "jobs-crud-close-job" }
+  - is_true: create_time
 
   - do:
       ml.open_job:
@@ -704,6 +802,7 @@
             }
           }
   - match: { job_id: "jobs-crud-close-a-closed-job" }
+  - is_true: create_time
 
   - do:
       ml.open_job:
@@ -738,6 +837,7 @@
             }
           }
   - match: { job_id: "jobs-crud-close-all-1" }
+  - is_true: create_time
 
   - do:
       ml.put_job:
@@ -756,6 +856,7 @@
             }
           }
   - match: { job_id: "jobs-crud-close-all-2" }
+  - is_true: create_time
 
   - do:
       ml.open_job:
@@ -892,6 +993,7 @@
             }
           }
   - match: { job_id: "jobs-crud-force-close-job" }
+  - is_true: create_time
 
   - do:
       ml.open_job:
@@ -936,6 +1038,7 @@
             }
           }
   - match: { job_id: "jobs-crud-close-a-closed-job" }
+  - is_true: create_time
 
   - do:
       ml.open_job:
@@ -1077,6 +1180,7 @@
   - is_true: datafeed_config
   - match: { datafeed_config.job_id: "jobs-crud-put-with-datafeed" }
   - match: { datafeed_config.datafeed_id: "jobs-crud-put-with-datafeed" }
+  - is_true: create_time
 
   - do:
       ml.get_datafeeds:
@@ -1104,6 +1208,8 @@
             }
           }
   - match: { job_id: "jobs-crud-get-with-datafeed" }
+  - is_true: create_time
+
   - do:
       ml.put_datafeed:
         datafeed_id: jobs-crud-get-with-datafeed
@@ -1157,6 +1263,7 @@
   - match: { job_id: "jobs-crud-put-with-datafeed-with-indices-options" }
   - match: { datafeed_config.datafeed_id: "jobs-crud-put-with-datafeed-with-indices-options" }
   - match: { datafeed_config.indices_options.ignore_throttled: false }
+  - is_true: create_time
 
   - do:
       ml.get_datafeeds:
@@ -1193,6 +1300,7 @@
           }
   - match: { job_id: "job-model-memory-limit-below-global-max" }
   - match: { analysis_limits.model_memory_limit: "8192mb" }
+  - is_true: create_time
 
   - do:
       catch: /model_memory_limit \[10gb\] must be less than the value of the xpack.ml.max_model_memory_limit setting \[9gb\]/
@@ -1235,6 +1343,7 @@
           }
   - match: { job_id: "job-model-memory-limit-above-removed-global-max" }
   - match: { analysis_limits.model_memory_limit: "10240mb" }
+  - is_true: create_time
 
 ---
 "Test jobs with named and custom categorization_analyzer":
@@ -1255,6 +1364,7 @@
           }
   - match: { job_id: "jobs-crud-named-categorization-analyzer-job" }
   - match: { analysis_config.categorization_analyzer: "standard" }
+  - is_true: create_time
 
   - do:
       ml.put_job:
@@ -1278,6 +1388,7 @@
   - match: { analysis_config.categorization_analyzer.char_filter.0: "html_strip" }
   - match: { analysis_config.categorization_analyzer.tokenizer: "classic" }
   - match: { analysis_config.categorization_analyzer.filter.0: "stop" }
+  - is_true: create_time
 
 ---
 "Test job with categorization_analyzer and categorization_filters":
@@ -1383,6 +1494,7 @@
           }
   - match: { job_id: "jobs-function-shortcut-expansion" }
   - match: { analysis_config.detectors.0.function: "non_zero_count"}
+  - is_true: create_time
 
 ---
 "Test open job when persistent task allocation disabled":
@@ -1412,6 +1524,7 @@
             }
           }
   - match: { job_id: "persistent-task-allocation-allowed-test" }
+  - is_true: create_time
 
   - do:
       catch: /Cannot open jobs because persistent task assignment is disabled by the \[cluster.persistent_tasks.allocation.enable\] setting/
@@ -1447,6 +1560,7 @@
             }
           }
   - match: { job_id: jobs-crud-reset-finished-time }
+  - is_true: create_time
 
   - do:
       ml.open_job:

x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/jobs_get_result_overall_buckets.yml

@@ -45,7 +45,8 @@ setup:
             "groups": [ "jobs-get-result-overall-buckets-group"],
             "analysis_config" : {
                 "bucket_span": "17m",
-                "detectors" :[{"function":"metric","field_name":"responsetime","by_field_name":"airline"}]
+                "detectors" :[{"function":"metric","field_name":"responsetime","by_field_name":"airline"}],
+                "model_prune_window":"34d"
             },
             "data_description" : {
                 "time_field":"time"