[ML] Automatic management for ML system indices (#68044)

The ML system indices now use the special functionality for
applying the correct mappings on first use. This replaces
the index templates that used to do this job, but were
vulnerable to tampering.

A number of other changes have had to be made to utilise
the system index functionality:

1. All fields previously missed out of mappings have been
   added to the system index mappings, with the types that
   would have been assigned dynamically in previous
   versions.  This is necessary because dynamic mappings
   updates are banned for system indices, yet some of our
   mappings allow dynamic updates.
2. As a result of the contradiction regarding dynamic
   mappings, we are now very well protected against failing
   to add new fields to the mappings for those indices that
   exhibit the contradiction (which are .ml-config and
   .ml-meta).  This means their mappings don't need to be
   explicitly compared to expected mappings in upgrade
   tests now.  Instead, any usage of a new field during or
   after upgrade will trigger an error in any test this occurs
   in.
3. Reserved fields for the config index were unnecessary
   (only used by tests) and just added extra complication,
   so they have been removed.  We have the concept of
   reserved fields for our results indices because end user
   fields get added to results and we need to ensure they
   don't clash with fields we want to use ourselves.  This
   problem does not exist for the config index.

server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java

@@ -172,7 +172,7 @@ public class TransportPutMappingAction extends AcknowledgedTransportMasterNodeAc
             return "Cannot update mappings in "
                 + violations
                 + ": system indices can only use mappings from their descriptors, "
-                + "but the mappings in the request did not match those in the descriptors(s)";
+                + "but the mappings in the request [" + requestMappings + "] did not match those in the descriptor(s)";
         }
 
         return null;

server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java

@@ -265,7 +265,7 @@ public class SystemIndexDescriptor {
         private String indexPattern;
         private String primaryIndex;
         private String description;
-        private XContentBuilder mappingsBuilder = null;
+        private String mappings = null;
         private Settings settings = null;
         private String aliasName = null;
         private int indexFormat = 0;
@@ -291,7 +291,12 @@ public class SystemIndexDescriptor {
         }
 
         public Builder setMappings(XContentBuilder mappingsBuilder) {
-            this.mappingsBuilder = mappingsBuilder;
+            mappings = mappingsBuilder == null ? null : Strings.toString(mappingsBuilder);
+            return this;
+        }
+
+        public Builder setMappings(String mappings) {
+            this.mappings = mappings;
             return this;
         }
 
@@ -330,7 +335,6 @@ public class SystemIndexDescriptor {
          * @return a populated descriptor.
          */
         public SystemIndexDescriptor build() {
-            String mappings = mappingsBuilder == null ? null : Strings.toString(mappingsBuilder);
 
             return new SystemIndexDescriptor(
                 indexPattern,

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigIndex.java

@@ -7,6 +7,9 @@
 package org.elasticsearch.xpack.core.ml;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.xpack.core.template.TemplateUtils;
 
 public final class MlConfigIndex {
@@ -14,6 +17,8 @@ public final class MlConfigIndex {
     private static final String INDEX_NAME = ".ml-config";
     private static final String MAPPINGS_VERSION_VARIABLE = "xpack.ml.version";
 
+    public static final int CONFIG_INDEX_MAX_RESULTS_WINDOW = 10_000;
+
     /**
      * The name of the index where job, datafeed and analytics configuration is stored
      *
@@ -30,5 +35,13 @@ public final class MlConfigIndex {
             MAPPINGS_VERSION_VARIABLE);
     }
 
+    public static Settings settings() {
+        return Settings.builder()
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
+            .put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), CONFIG_INDEX_MAX_RESULTS_WINDOW)
+            .build();
+    }
+
     private MlConfigIndex() {}
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java

@@ -6,9 +6,15 @@
  */
 package org.elasticsearch.xpack.core.ml;
 
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.xpack.core.template.TemplateUtils;
+
 public final class MlMetaIndex {
 
     private static final String INDEX_NAME = ".ml-meta";
+    private static final String MAPPINGS_VERSION_VARIABLE = "xpack.ml.version";
 
     /**
      * Where to store the ml info in Elasticsearch - must match what's
@@ -20,5 +26,19 @@ public final class MlMetaIndex {
         return INDEX_NAME;
     }
 
+    public static String mapping() {
+        return TemplateUtils.loadTemplate(
+            "/org/elasticsearch/xpack/core/ml/meta_index_mappings.json",
+            Version.CURRENT.toString(),
+            MAPPINGS_VERSION_VARIABLE);
+    }
+
+    public static Settings settings() {
+        return Settings.builder()
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
+            .build();
+    }
+
     private MlMetaIndex() {}
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/persistence/InferenceIndexConstants.java

@@ -6,7 +6,11 @@
  */
 package org.elasticsearch.xpack.core.ml.inference.persistence;
 
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.xpack.core.template.TemplateUtils;
 
 /**
  * Class containing the index constants so that the index version, name, and prefix are available to a wider audience.
@@ -29,5 +33,21 @@ public final class InferenceIndexConstants {
     public static final String LATEST_INDEX_NAME = INDEX_NAME_PREFIX + INDEX_VERSION;
     public static final ParseField DOC_TYPE = new ParseField("doc_type");
 
+    private static final String MAPPINGS_VERSION_VARIABLE = "xpack.ml.version";
+
+    public static String mapping() {
+        return TemplateUtils.loadTemplate(
+            "/org/elasticsearch/xpack/core/ml/inference_index_mappings.json",
+            Version.CURRENT.toString(),
+            MAPPINGS_VERSION_VARIABLE);
+    }
+
+    public static Settings settings() {
+        return Settings.builder()
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
+            .build();
+    }
+
     private InferenceIndexConstants() {}
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java

@@ -19,8 +19,6 @@ import org.elasticsearch.xpack.core.template.TemplateUtils;
  */
 public final class AnomalyDetectorsIndex {
 
-    public static final int CONFIG_INDEX_MAX_RESULTS_WINDOW = 10_000;
-
     private static final String RESULTS_MAPPINGS_VERSION_VARIABLE = "xpack.ml.version";
     private static final String RESOURCE_PATH = "/org/elasticsearch/xpack/core/ml/anomalydetection/";
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java

@@ -163,6 +163,7 @@ public class ElasticsearchMappings {
                 String mapping = mappingSupplier.get();
                 PutMappingRequest putMappingRequest = new PutMappingRequest(indicesThatRequireAnUpdate);
                 putMappingRequest.source(mapping, XContentType.JSON);
+                putMappingRequest.origin(ML_ORIGIN);
                 executeAsyncWithOrigin(client, ML_ORIGIN, PutMappingAction.INSTANCE, putMappingRequest,
                     ActionListener.wrap(response -> {
                         if (response.isAcknowledged()) {

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java

@@ -7,27 +7,9 @@
 package org.elasticsearch.xpack.core.ml.job.results;
 
 import org.elasticsearch.index.get.GetResult;
-import org.elasticsearch.xpack.core.ml.datafeed.ChunkingConfig;
-import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedTimingStats;
-import org.elasticsearch.xpack.core.ml.datafeed.DelayedDataCheckConfig;
-import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
-import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsDest;
-import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsSource;
-import org.elasticsearch.xpack.core.ml.dataframe.analyses.BoostedTreeParams;
-import org.elasticsearch.xpack.core.ml.dataframe.analyses.Classification;
-import org.elasticsearch.xpack.core.ml.dataframe.analyses.OutlierDetection;
-import org.elasticsearch.xpack.core.ml.dataframe.analyses.Regression;
-import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig;
-import org.elasticsearch.xpack.core.ml.job.config.AnalysisLimits;
-import org.elasticsearch.xpack.core.ml.job.config.DataDescription;
-import org.elasticsearch.xpack.core.ml.job.config.DetectionRule;
 import org.elasticsearch.xpack.core.ml.job.config.Detector;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
-import org.elasticsearch.xpack.core.ml.job.config.ModelPlotConfig;
-import org.elasticsearch.xpack.core.ml.job.config.Operator;
-import org.elasticsearch.xpack.core.ml.job.config.PerPartitionCategorizationConfig;
-import org.elasticsearch.xpack.core.ml.job.config.RuleCondition;
 import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats;
@@ -214,148 +196,6 @@ public final class ReservedFieldNames {
 
             GetResult._ID,
             GetResult._INDEX
-   };
-
-    /**
-     * This array should be updated to contain all the field names that appear
-     * in any documents we store in our config index.
-     */
-    private static final String[] RESERVED_CONFIG_FIELD_NAME_ARRAY = {
-            Job.ID.getPreferredName(),
-            Job.JOB_TYPE.getPreferredName(),
-            Job.JOB_VERSION.getPreferredName(),
-            Job.GROUPS.getPreferredName(),
-            Job.ANALYSIS_CONFIG.getPreferredName(),
-            Job.ANALYSIS_LIMITS.getPreferredName(),
-            Job.CREATE_TIME.getPreferredName(),
-            Job.CUSTOM_SETTINGS.getPreferredName(),
-            Job.DATA_DESCRIPTION.getPreferredName(),
-            Job.DESCRIPTION.getPreferredName(),
-            Job.FINISHED_TIME.getPreferredName(),
-            Job.MODEL_PLOT_CONFIG.getPreferredName(),
-            Job.RENORMALIZATION_WINDOW_DAYS.getPreferredName(),
-            Job.BACKGROUND_PERSIST_INTERVAL.getPreferredName(),
-            Job.MODEL_SNAPSHOT_RETENTION_DAYS.getPreferredName(),
-            Job.DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS.getPreferredName(),
-            Job.RESULTS_RETENTION_DAYS.getPreferredName(),
-            Job.MODEL_SNAPSHOT_ID.getPreferredName(),
-            Job.MODEL_SNAPSHOT_MIN_VERSION.getPreferredName(),
-            Job.RESULTS_INDEX_NAME.getPreferredName(),
-            Job.ALLOW_LAZY_OPEN.getPreferredName(),
-
-            AnalysisConfig.BUCKET_SPAN.getPreferredName(),
-            AnalysisConfig.CATEGORIZATION_FIELD_NAME.getPreferredName(),
-            AnalysisConfig.CATEGORIZATION_FILTERS.getPreferredName(),
-            AnalysisConfig.CATEGORIZATION_ANALYZER.getPreferredName(),
-            AnalysisConfig.PER_PARTITION_CATEGORIZATION.getPreferredName(),
-            AnalysisConfig.LATENCY.getPreferredName(),
-            AnalysisConfig.SUMMARY_COUNT_FIELD_NAME.getPreferredName(),
-            AnalysisConfig.DETECTORS.getPreferredName(),
-            AnalysisConfig.INFLUENCERS.getPreferredName(),
-            AnalysisConfig.MULTIVARIATE_BY_FIELDS.getPreferredName(),
-
-            AnalysisLimits.MODEL_MEMORY_LIMIT.getPreferredName(),
-            AnalysisLimits.CATEGORIZATION_EXAMPLES_LIMIT.getPreferredName(),
-
-            Detector.DETECTOR_DESCRIPTION_FIELD.getPreferredName(),
-            Detector.FUNCTION_FIELD.getPreferredName(),
-            Detector.FIELD_NAME_FIELD.getPreferredName(),
-            Detector.BY_FIELD_NAME_FIELD.getPreferredName(),
-            Detector.OVER_FIELD_NAME_FIELD.getPreferredName(),
-            Detector.PARTITION_FIELD_NAME_FIELD.getPreferredName(),
-            Detector.USE_NULL_FIELD.getPreferredName(),
-            Detector.EXCLUDE_FREQUENT_FIELD.getPreferredName(),
-            Detector.CUSTOM_RULES_FIELD.getPreferredName(),
-            Detector.DETECTOR_INDEX.getPreferredName(),
-
-            DetectionRule.ACTIONS_FIELD.getPreferredName(),
-            DetectionRule.CONDITIONS_FIELD.getPreferredName(),
-            DetectionRule.SCOPE_FIELD.getPreferredName(),
-            RuleCondition.APPLIES_TO_FIELD.getPreferredName(),
-            RuleCondition.VALUE_FIELD.getPreferredName(),
-            Operator.OPERATOR_FIELD.getPreferredName(),
-
-            DataDescription.FORMAT_FIELD.getPreferredName(),
-            DataDescription.TIME_FIELD_NAME_FIELD.getPreferredName(),
-            DataDescription.TIME_FORMAT_FIELD.getPreferredName(),
-            DataDescription.FIELD_DELIMITER_FIELD.getPreferredName(),
-            DataDescription.QUOTE_CHARACTER_FIELD.getPreferredName(),
-
-            ModelPlotConfig.ENABLED_FIELD.getPreferredName(),
-            ModelPlotConfig.TERMS_FIELD.getPreferredName(),
-            ModelPlotConfig.ANNOTATIONS_ENABLED_FIELD.getPreferredName(),
-
-            PerPartitionCategorizationConfig.STOP_ON_WARN.getPreferredName(),
-
-            DatafeedConfig.ID.getPreferredName(),
-            DatafeedConfig.QUERY_DELAY.getPreferredName(),
-            DatafeedConfig.FREQUENCY.getPreferredName(),
-            DatafeedConfig.INDICES.getPreferredName(),
-            DatafeedConfig.QUERY.getPreferredName(),
-            DatafeedConfig.SCROLL_SIZE.getPreferredName(),
-            DatafeedConfig.AGGREGATIONS.getPreferredName(),
-            DatafeedConfig.SCRIPT_FIELDS.getPreferredName(),
-            DatafeedConfig.CHUNKING_CONFIG.getPreferredName(),
-            DatafeedConfig.HEADERS.getPreferredName(),
-            DatafeedConfig.DELAYED_DATA_CHECK_CONFIG.getPreferredName(),
-            DatafeedConfig.INDICES_OPTIONS.getPreferredName(),
-            DelayedDataCheckConfig.ENABLED.getPreferredName(),
-            DelayedDataCheckConfig.CHECK_WINDOW.getPreferredName(),
-
-            ChunkingConfig.MODE_FIELD.getPreferredName(),
-            ChunkingConfig.TIME_SPAN_FIELD.getPreferredName(),
-
-            DataFrameAnalyticsConfig.ID.getPreferredName(),
-            DataFrameAnalyticsConfig.DESCRIPTION.getPreferredName(),
-            DataFrameAnalyticsConfig.SOURCE.getPreferredName(),
-            DataFrameAnalyticsConfig.DEST.getPreferredName(),
-            DataFrameAnalyticsConfig.ANALYSIS.getPreferredName(),
-            DataFrameAnalyticsConfig.ANALYZED_FIELDS.getPreferredName(),
-            DataFrameAnalyticsConfig.CREATE_TIME.getPreferredName(),
-            DataFrameAnalyticsConfig.VERSION.getPreferredName(),
-            DataFrameAnalyticsConfig.MAX_NUM_THREADS.getPreferredName(),
-            DataFrameAnalyticsDest.INDEX.getPreferredName(),
-            DataFrameAnalyticsDest.RESULTS_FIELD.getPreferredName(),
-            DataFrameAnalyticsSource.INDEX.getPreferredName(),
-            DataFrameAnalyticsSource.QUERY.getPreferredName(),
-            DataFrameAnalyticsSource._SOURCE.getPreferredName(),
-            OutlierDetection.NAME.getPreferredName(),
-            OutlierDetection.N_NEIGHBORS.getPreferredName(),
-            OutlierDetection.METHOD.getPreferredName(),
-            OutlierDetection.FEATURE_INFLUENCE_THRESHOLD.getPreferredName(),
-            Regression.NAME.getPreferredName(),
-            Regression.DEPENDENT_VARIABLE.getPreferredName(),
-            Regression.LOSS_FUNCTION.getPreferredName(),
-            Regression.LOSS_FUNCTION_PARAMETER.getPreferredName(),
-            Regression.PREDICTION_FIELD_NAME.getPreferredName(),
-            Regression.TRAINING_PERCENT.getPreferredName(),
-            Regression.FEATURE_PROCESSORS.getPreferredName(),
-            Regression.EARLY_STOPPING_ENABLED.getPreferredName(),
-            Classification.NAME.getPreferredName(),
-            Classification.DEPENDENT_VARIABLE.getPreferredName(),
-            Classification.PREDICTION_FIELD_NAME.getPreferredName(),
-            Classification.CLASS_ASSIGNMENT_OBJECTIVE.getPreferredName(),
-            Classification.NUM_TOP_CLASSES.getPreferredName(),
-            Classification.TRAINING_PERCENT.getPreferredName(),
-            Classification.FEATURE_PROCESSORS.getPreferredName(),
-            Classification.EARLY_STOPPING_ENABLED.getPreferredName(),
-            BoostedTreeParams.ALPHA.getPreferredName(),
-            BoostedTreeParams.DOWNSAMPLE_FACTOR.getPreferredName(),
-            BoostedTreeParams.LAMBDA.getPreferredName(),
-            BoostedTreeParams.GAMMA.getPreferredName(),
-            BoostedTreeParams.ETA.getPreferredName(),
-            BoostedTreeParams.ETA_GROWTH_RATE_PER_TREE.getPreferredName(),
-            BoostedTreeParams.MAX_OPTIMIZATION_ROUNDS_PER_HYPERPARAMETER.getPreferredName(),
-            BoostedTreeParams.MAX_TREES.getPreferredName(),
-            BoostedTreeParams.FEATURE_BAG_FRACTION.getPreferredName(),
-            BoostedTreeParams.NUM_TOP_FEATURE_IMPORTANCE_VALUES.getPreferredName(),
-            BoostedTreeParams.SOFT_TREE_DEPTH_LIMIT.getPreferredName(),
-            BoostedTreeParams.SOFT_TREE_DEPTH_TOLERANCE.getPreferredName(),
-
-            ElasticsearchMappings.CONFIG_TYPE,
-
-            GetResult._ID,
-            GetResult._INDEX,
     };
 
     /**
@@ -379,11 +219,6 @@ public final class ReservedFieldNames {
      */
     public static final Set<String> RESERVED_RESULT_FIELD_NAMES = new HashSet<>(Arrays.asList(RESERVED_RESULT_FIELD_NAME_ARRAY));
 
-    /**
-     * A set of all reserved field names in our config.
-     */
-    public static final Set<String> RESERVED_CONFIG_FIELD_NAMES = new HashSet<>(Arrays.asList(RESERVED_CONFIG_FIELD_NAME_ARRAY));
-
     private ReservedFieldNames() {
     }
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java

@@ -30,6 +30,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.indices.SystemIndexDescriptor;
 import org.elasticsearch.xpack.core.template.IndexTemplateConfig;
 
 import java.util.Arrays;
@@ -162,6 +163,48 @@ public final class MlIndexAndAlias {
         loggingListener.onResponse(false);
     }
 
+    public static void createSystemIndexIfNecessary(Client client,
+                                                    ClusterState clusterState,
+                                                    SystemIndexDescriptor descriptor,
+                                                    ActionListener<Boolean> finalListener) {
+
+        final String primaryIndex = descriptor.getPrimaryIndex();
+
+        // The check for existence of the index is against the cluster state, so very cheap
+        if (hasIndex(clusterState, primaryIndex)) {
+            finalListener.onResponse(true);
+            return;
+        }
+
+        ActionListener<Boolean> indexCreatedListener = ActionListener.wrap(
+            created -> {
+                if (created) {
+                    waitForShardsReady(client, primaryIndex, finalListener);
+                } else {
+                    finalListener.onResponse(false);
+                }
+            },
+            e -> {
+                if (ExceptionsHelper.unwrapCause(e) instanceof ResourceAlreadyExistsException) {
+                    finalListener.onResponse(true);
+                } else {
+                    finalListener.onFailure(e);
+                }
+            }
+        );
+
+        CreateIndexRequest createIndexRequest = new CreateIndexRequest(primaryIndex);
+        createIndexRequest.settings(descriptor.getSettings());
+        createIndexRequest.mapping(descriptor.getMappings());
+        createIndexRequest.origin(ML_ORIGIN);
+
+        executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, createIndexRequest,
+            ActionListener.<CreateIndexResponse>wrap(
+                r -> indexCreatedListener.onResponse(r.isAcknowledged()),
+                indexCreatedListener::onFailure
+            ), client.admin().indices()::create);
+    }
+
     private static void waitForShardsReady(Client client, String index, ActionListener<Boolean> listener) {
         ClusterHealthRequest healthRequest = Requests.clusterHealthRequest(index)
             .waitForYellowStatus()
@@ -309,4 +352,8 @@ public final class MlIndexAndAlias {
     public static boolean hasIndexTemplate(ClusterState state, String templateName) {
         return state.getMetadata().getTemplates().containsKey(templateName);
     }
+
+    public static boolean hasIndex(ClusterState state, String index) {
+        return state.getMetadata().getIndicesLookup().containsKey(index);
+    }
 }

x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_mappings.json

@@ -21,6 +21,9 @@
       "allow_lazy_open" : {
         "type" : "keyword"
       },
+      "allow_lazy_start" : {
+        "type" : "keyword"
+      },
       "analysis" : {
         "properties" : {
           "classification" : {
@@ -34,6 +37,9 @@
               "downsample_factor" : {
                 "type" : "double"
               },
+              "early_stopping_enabled" : {
+                "type": "boolean"
+              },
               "eta" : {
                 "type" : "double"
               },
@@ -70,6 +76,9 @@
               "prediction_field_name" : {
                 "type" : "keyword"
               },
+              "randomize_seed" : {
+                "type" : "keyword"
+              },
               "soft_tree_depth_limit" : {
                 "type" : "double"
               },
@@ -78,22 +87,28 @@
               },
               "training_percent" : {
                 "type" : "double"
-              },
-              "early_stopping_enabled" : {
-                "type": "boolean"
               }
             }
           },
           "outlier_detection" : {
             "properties" : {
+              "compute_feature_influence" : {
+                "type" : "keyword"
+              },
               "feature_influence_threshold" : {
                 "type" : "double"
               },
               "method" : {
                 "type" : "keyword"
               },
+              "outlier_fraction" : {
+                "type" : "keyword"
+              },
               "n_neighbors" : {
                 "type" : "integer"
+              },
+              "standardization_enabled" : {
+                "type" : "keyword"
               }
             }
           },
@@ -108,6 +123,9 @@
               "downsample_factor" : {
                 "type" : "double"
               },
+              "early_stopping_enabled" : {
+                "type": "boolean"
+              },
               "eta" : {
                 "type" : "double"
               },
@@ -144,6 +162,9 @@
               "prediction_field_name" : {
                 "type" : "keyword"
               },
+              "randomize_seed" : {
+                "type" : "keyword"
+              },
               "soft_tree_depth_limit" : {
                 "type" : "double"
               },
@@ -152,9 +173,6 @@
               },
               "training_percent" : {
                 "type" : "double"
-              },
-              "early_stopping_enabled" : {
-                "type": "boolean"
               }
             }
           }
@@ -328,6 +346,9 @@
           }
         }
       },
+      "deleting" : {
+        "type" : "keyword"
+      },
       "description" : {
         "type" : "text"
       },
@@ -373,6 +394,9 @@
       "job_version" : {
         "type" : "keyword"
       },
+      "max_empty_searches" : {
+        "type" : "keyword"
+      },
       "max_num_threads" : {
         "type" : "integer"
       },

x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_template.json

@@ -1,15 +0,0 @@
-{
-  "order" : 0,
-  "version" : ${xpack.ml.version.id},
-  "index_patterns" : [
-    ".ml-config"
-  ],
-  "settings" : {
-    "index" : {
-      "max_result_window" : "${xpack.ml.config.max_result_window}",
-      "number_of_shards" : "1",
-      "auto_expand_replicas" : "0-1"
-    }
-  },
-  "mappings": ${xpack.ml.config.mappings}
-}

x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/inference_index_mappings.json

@@ -0,0 +1,142 @@
+{
+  "_doc": {
+    "_meta": {
+      "version": "${xpack.ml.version}"
+    },
+    "dynamic": false,
+    "properties": {
+      "doc_type": {
+        "type": "keyword"
+      },
+      "model_id": {
+        "type": "keyword"
+      },
+      "created_by": {
+        "type": "keyword"
+      },
+      "input": {
+        "enabled": false
+      },
+      "version": {
+        "type": "keyword"
+      },
+      "description": {
+        "type": "text"
+      },
+      "create_time": {
+        "type": "date"
+      },
+      "tags": {
+        "type": "keyword"
+      },
+      "metadata": {
+        "enabled": false
+      },
+      "estimated_operations": {
+        "type": "long"
+      },
+      "estimated_heap_memory_usage_bytes": {
+        "type": "long"
+      },
+      "doc_num": {
+        "type": "long"
+      },
+      "definition": {
+        "enabled": false
+      },
+      "compression_version": {
+        "type": "long"
+      },
+      "definition_length": {
+        "type": "long"
+      },
+      "total_definition_length": {
+        "type": "long"
+      },
+      "default_field_map": {
+        "enabled": false
+      },
+      "inference_config": {
+        "enabled": false
+      },
+      "feature_importance_baseline": {
+        "properties": {
+          "baseline": {
+            "type": "double"
+          },
+          "classes": {
+            "properties": {
+              "class_name": { "type": "keyword"},
+              "baseline": {"type" : "double"}
+            }
+          }
+        }
+      },
+      "total_feature_importance": {
+        "type": "nested",
+        "dynamic": false,
+        "properties": {
+          "importance": {
+            "properties": {
+              "min": {
+                "type": "double"
+              },
+              "max": {
+                "type": "double"
+              },
+              "mean_magnitude": {
+                "type": "double"
+              }
+            }
+          },
+          "feature_name": {
+            "type": "keyword"
+          },
+          "classes": {
+            "type": "nested",
+            "dynamic": false,
+            "properties": {
+              "importance": {
+                "properties": {
+                  "min": {
+                    "type": "double"
+                  },
+                  "max": {
+                    "type": "double"
+                  },
+                  "mean_magnitude": {
+                    "type": "double"
+                  }
+                }
+              },
+              "class_name": {
+                "type": "keyword"
+              }
+            }
+          }
+        }
+      },
+      "hyperparameters": {
+        "type": "nested",
+        "dynamic": false,
+        "properties": {
+          "name": {
+            "type": "keyword"
+          },
+          "value": {
+            "type": "double"
+          },
+          "absolute_importance": {
+            "type": "double"
+          },
+          "relative_importance": {
+            "type": "double"
+          },
+          "supplied": {
+            "type": "boolean"
+          }
+        } 
+      }
+    }
+  }
+}

x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/inference_index_template.json

@@ -1,156 +0,0 @@
-{
-  "order" : 0,
-  "version" : ${xpack.ml.version.id},
-  "index_patterns" : [
-    ".ml-inference-000003"
-  ],
-  "settings" : {
-    "index" : {
-      "number_of_shards" : "1",
-      "auto_expand_replicas" : "0-1"
-    }
-  },
-  "mappings" : {
-    "_doc": {
-      "_meta": {
-        "version": "${xpack.ml.version}"
-      },
-      "dynamic": "false",
-      "properties": {
-        "doc_type": {
-          "type": "keyword"
-        },
-        "model_id": {
-          "type": "keyword"
-        },
-        "created_by": {
-          "type": "keyword"
-        },
-        "input": {
-          "enabled": false
-        },
-        "version": {
-          "type": "keyword"
-        },
-        "description": {
-          "type": "text"
-        },
-        "create_time": {
-          "type": "date"
-        },
-        "tags": {
-          "type": "keyword"
-        },
-        "metadata": {
-          "enabled": false
-        },
-        "estimated_operations": {
-          "type": "long"
-        },
-        "estimated_heap_memory_usage_bytes": {
-          "type": "long"
-        },
-        "doc_num": {
-          "type": "long"
-        },
-        "definition": {
-          "enabled": false
-        },
-        "compression_version": {
-          "type": "long"
-        },
-        "definition_length": {
-          "type": "long"
-        },
-        "total_definition_length": {
-          "type": "long"
-        },
-        "default_field_map": {
-          "enabled": false
-        },
-        "inference_config": {
-          "enabled": false
-        },
-        "feature_importance_baseline": {
-          "properties": {
-            "baseline": {
-              "type": "double"
-            },
-            "classes": {
-              "properties": {
-                "class_name": { "type": "keyword"},
-                "baseline": {"type" : "double"}
-              }
-            }
-          }
-        },
-        "total_feature_importance": {
-          "type": "nested",
-          "dynamic": "false",
-          "properties": {
-            "importance": {
-              "properties": {
-                "min": {
-                  "type": "double"
-                },
-                "max": {
-                  "type": "double"
-                },
-                "mean_magnitude": {
-                  "type": "double"
-                }
-              }
-            },
-            "feature_name": {
-              "type": "keyword"
-            },
-            "classes": {
-              "type": "nested",
-              "dynamic": "false",
-              "properties": {
-                "importance": {
-                  "properties": {
-                    "min": {
-                      "type": "double"
-                    },
-                    "max": {
-                      "type": "double"
-                    },
-                    "mean_magnitude": {
-                      "type": "double"
-                    }
-                  }
-                },
-                "class_name": {
-                  "type": "keyword"
-                }
-              }
-            }
-          }
-        },
-        "hyperparameters": {
-          "type": "nested",
-          "dynamic": "false",
-          "properties": {
-            "name": {
-              "type": "keyword"
-            },
-            "value": {
-              "type": "double"
-            },
-            "absolute_importance": {
-              "type": "double"
-            },
-            "relative_importance": {
-              "type": "double"
-            },
-            "supplied": {
-              "type": "boolean"
-            }
-          } 
-        }
-      }
-    }
-  },
-  "aliases" : { }
-}

x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/meta_index_mappings.json

@@ -0,0 +1,43 @@
+{
+  "_doc": {
+    "_meta": {
+      "version": "${xpack.ml.version}"
+    },
+    "dynamic_templates": [
+      {
+        "strings_as_keywords": {
+          "match": "*",
+          "mapping": {
+            "type": "keyword"
+          }
+        }
+      }
+    ],
+    "properties": {
+      "calendar_id": {
+        "type": "keyword"
+      },
+      "job_ids": {
+        "type": "keyword"
+      },
+      "description": {
+        "type": "keyword"
+      },
+      "filter_id": {
+        "type": "keyword"
+      },
+      "items": {
+        "type": "keyword"
+      },
+      "start_time": {
+        "type": "date"
+      },
+      "end_time": {
+        "type": "date"
+      },
+      "type": {
+        "type": "keyword"
+      }
+    }
+  }
+}

x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/meta_index_template.json

@@ -1,47 +0,0 @@
-{
-  "order" : 0,
-  "version" : ${xpack.ml.version.id},
-  "index_patterns" : [
-    ".ml-meta"
-  ],
-  "settings" : {
-    "index" : {
-      "number_of_shards" : "1",
-      "auto_expand_replicas" : "0-1"
-    }
-  },
-  "mappings" : {
-    "_doc": {
-      "_meta": {
-        "version": "${xpack.ml.version}"
-      },
-      "dynamic_templates": [
-        {
-          "strings_as_keywords": {
-            "match": "*",
-            "mapping": {
-              "type": "keyword"
-            }
-          }
-        }
-      ],
-      "properties": {
-        "calendar_id": {
-          "type": "keyword"
-        },
-        "job_ids": {
-          "type": "keyword"
-        },
-        "description": {
-          "type": "keyword"
-        },
-        "start_time": {
-          "type": "date"
-        },
-        "end_time": {
-          "type": "date"
-        }
-      }
-    }
-  }
-}

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java

@@ -28,11 +28,7 @@ import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.xpack.core.ml.MlConfigIndex;
-import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedTimingStats;
-import org.elasticsearch.xpack.core.ml.job.config.Job;
-import org.elasticsearch.xpack.core.ml.job.config.ModelPlotConfig;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot;
@@ -109,22 +105,6 @@ public class ElasticsearchMappingsTests extends ESTestCase {
         compareFields(expected, ReservedFieldNames.RESERVED_RESULT_FIELD_NAMES);
     }
 
-    public void testConfigMappingReservedFields() throws Exception {
-        Set<String> overridden = new HashSet<>(KEYWORDS);
-
-        // These are not reserved because they're data types, not field names
-        overridden.add(Job.TYPE);
-        overridden.add(DatafeedConfig.TYPE);
-        // ModelPlotConfig has an 'enabled' the same as one of the keywords
-        overridden.remove(ModelPlotConfig.ENABLED_FIELD.getPreferredName());
-
-        Set<String> expected = collectConfigDocFieldNames();
-        expected.removeAll(overridden);
-        expected.addAll(INTERNAL_FIELDS);
-
-        compareFields(expected, ReservedFieldNames.RESERVED_CONFIG_FIELD_NAMES);
-    }
-
     private void compareFields(Set<String> expected, Set<String> reserved) {
         if (reserved.size() != expected.size()) {
             Set<String> diff = new HashSet<>(reserved);
@@ -145,7 +125,7 @@ public class ElasticsearchMappingsTests extends ESTestCase {
         }
     }
 
-    public void testMappingRequiresUpdateNoMapping() throws IOException {
+    public void testMappingRequiresUpdateNoMapping() {
         ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name"));
         ClusterState cs = csBuilder.build();
         String[] indices = new String[] { "no_index" };
@@ -153,44 +133,44 @@ public class ElasticsearchMappingsTests extends ESTestCase {
         assertArrayEquals(new String[] { "no_index" }, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, Version.CURRENT));
     }
 
-    public void testMappingRequiresUpdateNullMapping() throws IOException {
+    public void testMappingRequiresUpdateNullMapping() {
         ClusterState cs = getClusterStateWithMappingsWithMetadata(Collections.singletonMap("null_mapping", null));
         String[] indices = new String[] { "null_index" };
         assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, Version.CURRENT));
     }
 
-    public void testMappingRequiresUpdateNoVersion() throws IOException {
+    public void testMappingRequiresUpdateNoVersion() {
         ClusterState cs = getClusterStateWithMappingsWithMetadata(Collections.singletonMap("no_version_field", "NO_VERSION_FIELD"));
         String[] indices = new String[] { "no_version_field" };
         assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, Version.CURRENT));
     }
 
-    public void testMappingRequiresUpdateRecentMappingVersion() throws IOException {
+    public void testMappingRequiresUpdateRecentMappingVersion() {
         ClusterState cs = getClusterStateWithMappingsWithMetadata(Collections.singletonMap("version_current", Version.CURRENT.toString()));
         String[] indices = new String[] { "version_current" };
         assertArrayEquals(new String[] {}, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, Version.CURRENT));
     }
 
-    public void testMappingRequiresUpdateMaliciousMappingVersion() throws IOException {
+    public void testMappingRequiresUpdateMaliciousMappingVersion() {
         ClusterState cs = getClusterStateWithMappingsWithMetadata(
             Collections.singletonMap("version_current", Collections.singletonMap("nested", "1.0")));
         String[] indices = new String[] { "version_nested" };
         assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, Version.CURRENT));
     }
 
-    public void testMappingRequiresUpdateBogusMappingVersion() throws IOException {
+    public void testMappingRequiresUpdateBogusMappingVersion() {
         ClusterState cs = getClusterStateWithMappingsWithMetadata(Collections.singletonMap("version_bogus", "0.0"));
         String[] indices = new String[] { "version_bogus" };
         assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, Version.CURRENT));
     }
 
-    public void testMappingRequiresUpdateNewerMappingVersion() throws IOException {
+    public void testMappingRequiresUpdateNewerMappingVersion() {
         ClusterState cs = getClusterStateWithMappingsWithMetadata(Collections.singletonMap("version_newer", Version.CURRENT));
         String[] indices = new String[] { "version_newer" };
         assertArrayEquals(new String[] {}, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, VersionUtils.getPreviousVersion()));
     }
 
-    public void testMappingRequiresUpdateNewerMappingVersionMinor() throws IOException {
+    public void testMappingRequiresUpdateNewerMappingVersionMinor() {
         ClusterState cs = getClusterStateWithMappingsWithMetadata(Collections.singletonMap("version_newer_minor", Version.CURRENT));
         String[] indices = new String[] { "version_newer_minor" };
         assertArrayEquals(new String[] {},
@@ -274,11 +254,6 @@ public class ElasticsearchMappingsTests extends ESTestCase {
         return collectFieldNames(AnomalyDetectorsIndex.resultsMapping());
     }
 
-    private Set<String> collectConfigDocFieldNames() throws IOException {
-        // Only the mappings for the config index should be added below.  Do NOT add mappings for other indexes here.
-        return collectFieldNames(MlConfigIndex.mapping());
-    }
-
     private Set<String> collectFieldNames(String mapping) throws IOException {
         BufferedInputStream inputStream =
                 new BufferedInputStream(new ByteArrayInputStream(mapping.getBytes(StandardCharsets.UTF_8)));

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java

@@ -139,11 +139,11 @@ public class MlIndexAndAliasTests extends ESTestCase {
     public void testInstallIndexTemplateIfRequired() {
         ClusterState clusterState = createClusterState(Collections.emptyMap());
 
-        IndexTemplateConfig inferenceTemplate = new IndexTemplateConfig(InferenceIndexConstants.LATEST_INDEX_NAME,
-            "/org/elasticsearch/xpack/core/ml/inference_index_template.json", Version.CURRENT.id, "xpack.ml.version",
+        IndexTemplateConfig notificationsTemplate = new IndexTemplateConfig(InferenceIndexConstants.LATEST_INDEX_NAME,
+            "/org/elasticsearch/xpack/core/ml/notifications_index_template.json", Version.CURRENT.id, "xpack.ml.version",
             Collections.singletonMap("xpack.ml.version.id", String.valueOf(Version.CURRENT.id)));
 
-        MlIndexAndAlias.installIndexTemplateIfRequired(clusterState, client, inferenceTemplate, listener);
+        MlIndexAndAlias.installIndexTemplateIfRequired(clusterState, client, notificationsTemplate, listener);
         InOrder inOrder = inOrder(indicesAdminClient, listener);
         inOrder.verify(indicesAdminClient).putTemplate(any(), any());
         inOrder.verify(listener).onResponse(true);

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java

@@ -70,7 +70,6 @@ import org.elasticsearch.threadpool.ScalingExecutorBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.watcher.ResourceWatcherService;
 import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingDeciderService;
-import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.XPackPlugin;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
@@ -359,6 +358,7 @@ import java.util.function.UnaryOperator;
 
 import static java.util.Collections.emptyList;
 import static java.util.Collections.singletonList;
+import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
 
 public class MachineLearning extends Plugin implements SystemIndexPlugin,
                                                        AnalysisPlugin,
@@ -637,7 +637,7 @@ public class MachineLearning extends Plugin implements SystemIndexPlugin,
         DataFrameAnalyticsAuditor dataFrameAnalyticsAuditor = new DataFrameAnalyticsAuditor(client, clusterService);
         InferenceAuditor inferenceAuditor = new InferenceAuditor(client, clusterService);
         this.dataFrameAnalyticsAuditor.set(dataFrameAnalyticsAuditor);
-        OriginSettingClient originSettingClient = new OriginSettingClient(client, ClientHelper.ML_ORIGIN);
+        OriginSettingClient originSettingClient = new OriginSettingClient(client, ML_ORIGIN);
         ResultsPersisterService resultsPersisterService = new ResultsPersisterService(
             threadPool,
             originSettingClient,
@@ -854,8 +854,7 @@ public class MachineLearning extends Plugin implements SystemIndexPlugin,
                     dataFrameAnalyticsManager.get(),
                     dataFrameAnalyticsAuditor.get(),
                     memoryTracker.get(),
-                    expressionResolver,
-                    MlIndexTemplateRegistry.INFERENCE_TEMPLATE),
+                    expressionResolver),
                 new SnapshotUpgradeTaskExecutor(settings,
                     clusterService,
                     autodetectProcessManager.get(),
@@ -1085,10 +1084,8 @@ public class MachineLearning extends Plugin implements SystemIndexPlugin,
         List<String> templateNames =
             Arrays.asList(
                 NotificationsIndex.NOTIFICATIONS_INDEX,
-                MlMetaIndex.indexName(),
                 AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX,
-                AnomalyDetectorsIndex.jobResultsIndexPrefix(),
-                InferenceIndexConstants.LATEST_INDEX_NAME);
+                AnomalyDetectorsIndex.jobResultsIndexPrefix());
         for (String templateName : templateNames) {
             allPresent = allPresent && TemplateUtils.checkTemplateExistsAndVersionIsGTECurrentVersion(templateName, clusterState);
         }
@@ -1165,12 +1162,40 @@ public class MachineLearning extends Plugin implements SystemIndexPlugin,
     @Override
     public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
         return List.of(
-            new SystemIndexDescriptor(MlMetaIndex.indexName(), "Contains scheduling and anomaly tracking metadata"),
-            new SystemIndexDescriptor(MlConfigIndex.indexName(), "Contains ML configuration data"),
-            new SystemIndexDescriptor(InferenceIndexConstants.INDEX_PATTERN, "Contains ML model configuration and statistics")
+            SystemIndexDescriptor.builder()
+                .setIndexPattern(MlMetaIndex.indexName() + "*")
+                .setPrimaryIndex(MlMetaIndex.indexName())
+                .setDescription("Contains scheduling and anomaly tracking metadata")
+                .setMappings(MlMetaIndex.mapping())
+                .setSettings(MlMetaIndex.settings())
+                .setVersionMetaKey("version")
+                .setOrigin(ML_ORIGIN)
+                .build(),
+            SystemIndexDescriptor.builder()
+                .setIndexPattern(MlConfigIndex.indexName() + "*")
+                .setPrimaryIndex(MlConfigIndex.indexName())
+                .setDescription("Contains ML configuration data")
+                .setMappings(MlConfigIndex.mapping())
+                .setSettings(MlConfigIndex.settings())
+                .setVersionMetaKey("version")
+                .setOrigin(ML_ORIGIN)
+                .build(),
+            getInferenceIndexSecurityDescriptor()
         );
     }
 
+    public static SystemIndexDescriptor getInferenceIndexSecurityDescriptor() {
+        return SystemIndexDescriptor.builder()
+            .setIndexPattern(InferenceIndexConstants.INDEX_PATTERN)
+            .setPrimaryIndex(InferenceIndexConstants.LATEST_INDEX_NAME)
+            .setDescription("Contains ML model configuration and statistics")
+            .setMappings(InferenceIndexConstants.mapping())
+            .setSettings(InferenceIndexConstants.settings())
+            .setVersionMetaKey("version")
+            .setOrigin(ML_ORIGIN)
+            .build();
+    }
+
     @Override
     public BreakerSettings getCircuitBreaker(Settings settings) {
         return BreakerSettings.updateFromSettings(

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java

@@ -23,7 +23,6 @@ import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
-import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -34,7 +33,6 @@ import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.xpack.core.ml.MlConfigIndex;
@@ -492,13 +490,9 @@ public class MlConfigMigrator {
         CreateIndexRequest createIndexRequest = new CreateIndexRequest(MlConfigIndex.indexName());
         try
         {
-            createIndexRequest.settings(
-                    Settings.builder()
-                            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
-                            .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
-                            .put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), AnomalyDetectorsIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW)
-            );
+            createIndexRequest.settings(MlConfigIndex.settings());
             createIndexRequest.mapping(MlConfigIndex.mapping());
+            createIndexRequest.origin(ML_ORIGIN);
         } catch (Exception e) {
             logger.error("error writing the .ml-config mappings", e);
             listener.onFailure(e);

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java

@@ -13,10 +13,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.ClientHelper;
-import org.elasticsearch.xpack.core.ml.MlConfigIndex;
-import org.elasticsearch.xpack.core.ml.MlMetaIndex;
 import org.elasticsearch.xpack.core.ml.MlStatsIndex;
-import org.elasticsearch.xpack.core.ml.inference.persistence.InferenceIndexConstants;
 import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
 import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields;
 import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex;
@@ -43,39 +40,16 @@ public class MlIndexTemplateRegistry extends IndexTemplateRegistry {
 
     private static final IndexTemplateConfig ANOMALY_DETECTION_STATE_TEMPLATE = stateTemplate();
 
-    private static final IndexTemplateConfig META_TEMPLATE = new IndexTemplateConfig(MlMetaIndex.indexName(),
-        ROOT_RESOURCE_PATH + "meta_index_template.json", Version.CURRENT.id, VERSION_PATTERN,
-        Collections.singletonMap(VERSION_ID_PATTERN, String.valueOf(Version.CURRENT.id)));
-
     public static final IndexTemplateConfig NOTIFICATIONS_TEMPLATE = new IndexTemplateConfig(NotificationsIndex.NOTIFICATIONS_INDEX,
         ROOT_RESOURCE_PATH + "notifications_index_template.json", Version.CURRENT.id, VERSION_PATTERN,
         Collections.singletonMap(VERSION_ID_PATTERN, String.valueOf(Version.CURRENT.id)));
 
-    private static final IndexTemplateConfig CONFIG_TEMPLATE = configTemplate();
-
-    public static final IndexTemplateConfig INFERENCE_TEMPLATE = new IndexTemplateConfig(InferenceIndexConstants.LATEST_INDEX_NAME,
-        ROOT_RESOURCE_PATH + "inference_index_template.json", Version.CURRENT.id, VERSION_PATTERN,
-        Collections.singletonMap(VERSION_ID_PATTERN, String.valueOf(Version.CURRENT.id)));
-
     private static final IndexTemplateConfig STATS_TEMPLATE = statsTemplate();
 
     private static final String ML_SIZE_BASED_ILM_POLICY_NAME = "ml-size-based-ilm-policy";
     private static final LifecyclePolicyConfig ML_SIZE_BASED_ILM_POLICY =
         new LifecyclePolicyConfig(ML_SIZE_BASED_ILM_POLICY_NAME, ROOT_RESOURCE_PATH + "size_based_ilm_policy.json");
 
-    private static IndexTemplateConfig configTemplate() {
-        Map<String, String> variables = new HashMap<>();
-        variables.put(VERSION_ID_PATTERN, String.valueOf(Version.CURRENT.id));
-        variables.put("xpack.ml.config.max_result_window",
-            String.valueOf(AnomalyDetectorsIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW));
-        variables.put("xpack.ml.config.mappings", MlConfigIndex.mapping());
-
-        return new IndexTemplateConfig(MlConfigIndex.indexName(),
-            ROOT_RESOURCE_PATH + "config_index_template.json",
-            Version.CURRENT.id, VERSION_PATTERN,
-            variables);
-    }
-
     private static IndexTemplateConfig stateTemplate() {
         Map<String, String> variables = new HashMap<>();
         variables.put(VERSION_ID_PATTERN, String.valueOf(Version.CURRENT.id));
@@ -120,9 +94,6 @@ public class MlIndexTemplateRegistry extends IndexTemplateRegistry {
         templatesToUse = Arrays.asList(
             ANOMALY_DETECTION_RESULTS_TEMPLATE,
             ANOMALY_DETECTION_STATE_TEMPLATE,
-            CONFIG_TEMPLATE,
-            INFERENCE_TEMPLATE,
-            META_TEMPLATE,
             NOTIFICATIONS_TEMPLATE,
             STATS_TEMPLATE);
     }

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java

@@ -61,12 +61,12 @@ import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsTaskState;
 import org.elasticsearch.xpack.core.ml.dataframe.analyses.RequiredField;
+import org.elasticsearch.xpack.core.ml.inference.persistence.InferenceIndexConstants;
 import org.elasticsearch.xpack.core.ml.job.messages.Messages;
 import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.core.ml.utils.MlIndexAndAlias;
 import org.elasticsearch.xpack.core.ml.utils.PhaseProgress;
-import org.elasticsearch.xpack.core.template.IndexTemplateConfig;
 import org.elasticsearch.xpack.ml.MachineLearning;
 import org.elasticsearch.xpack.ml.dataframe.DataFrameAnalyticsManager;
 import org.elasticsearch.xpack.ml.dataframe.DataFrameAnalyticsTask;
@@ -585,13 +585,11 @@ public class TransportStartDataFrameAnalyticsAction
         private final Client client;
         private final DataFrameAnalyticsManager manager;
         private final DataFrameAnalyticsAuditor auditor;
-        private final IndexTemplateConfig inferenceIndexTemplate;
 
         private volatile ClusterState clusterState;
 
         public TaskExecutor(Settings settings, Client client, ClusterService clusterService, DataFrameAnalyticsManager manager,
-                            DataFrameAnalyticsAuditor auditor, MlMemoryTracker memoryTracker, IndexNameExpressionResolver resolver,
-                            IndexTemplateConfig inferenceIndexTemplate) {
+                            DataFrameAnalyticsAuditor auditor, MlMemoryTracker memoryTracker, IndexNameExpressionResolver resolver) {
             super(MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME,
                 MachineLearning.UTILITY_THREAD_POOL_NAME,
                 settings,
@@ -601,7 +599,6 @@ public class TransportStartDataFrameAnalyticsAction
             this.client = Objects.requireNonNull(client);
             this.manager = Objects.requireNonNull(manager);
             this.auditor = Objects.requireNonNull(auditor);
-            this.inferenceIndexTemplate = Objects.requireNonNull(inferenceIndexTemplate);
             clusterService.addListener(event -> clusterState = event.state());
         }
 
@@ -674,22 +671,26 @@ public class TransportStartDataFrameAnalyticsAction
             );
 
             // Get stats to initialize in memory stats tracking
-            ActionListener<Boolean> templateCheckListener = ActionListener.wrap(
+            ActionListener<Boolean> indexCheckListener = ActionListener.wrap(
                 ok -> executeAsyncWithOrigin(client, ML_ORIGIN, GetDataFrameAnalyticsStatsAction.INSTANCE,
                     new GetDataFrameAnalyticsStatsAction.Request(params.getId()), statsListener),
                 error -> {
                     Throwable cause = ExceptionsHelper.unwrapCause(error);
                     logger.error(
                         new ParameterizedMessage(
-                            "[{}] failed to create internal index template [{}]",
+                            "[{}] failed to create internal index [{}]",
                             params.getId(),
-                            inferenceIndexTemplate.getTemplateName()),
+                            InferenceIndexConstants.LATEST_INDEX_NAME),
                         cause);
                     dfaTask.setFailed(error);
                 }
             );
 
-            MlIndexAndAlias.installIndexTemplateIfRequired(clusterState, client, inferenceIndexTemplate, templateCheckListener);
+            // Create the system index explicitly.  Although the master node would create it automatically on first use,
+            // in a mixed version cluster where the master node is on an older version than this node relying on auto-creation
+            // might use outdated mappings.
+            MlIndexAndAlias.createSystemIndexIfNecessary(client, clusterState, MachineLearning.getInferenceIndexSecurityDescriptor(),
+                indexCheckListener);
         }
 
         private void executeTask(DataFrameAnalyticsTask task) {

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java

@@ -52,7 +52,6 @@ import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedUpdate;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
-import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.core.ml.utils.MlStrings;
 import org.elasticsearch.xpack.core.ml.utils.ToXContentParams;
@@ -78,7 +77,7 @@ import static org.elasticsearch.xpack.core.ClientHelper.filterSecurityHeaders;
  * datafeed configuration document
  *
  * The number of datafeeds returned in a search it limited to
- * {@link AnomalyDetectorsIndex#CONFIG_INDEX_MAX_RESULTS_WINDOW}.
+ * {@link MlConfigIndex#CONFIG_INDEX_MAX_RESULTS_WINDOW}.
  * In most cases we expect 10s or 100s of datafeeds to be defined and
  * a search for all datafeeds should return all.
  */
@@ -369,7 +368,7 @@ public class DatafeedConfigProvider {
         SearchRequest searchRequest = client.prepareSearch(MlConfigIndex.indexName())
                 .setIndicesOptions(IndicesOptions.lenientExpandOpen())
                 .setSource(sourceBuilder)
-                .setSize(AnomalyDetectorsIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW)
+                .setSize(MlConfigIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW)
                 .request();
 
         ExpandedIdsMatcher requiredMatches = new ExpandedIdsMatcher(tokens, allowNoMatch);
@@ -421,7 +420,7 @@ public class DatafeedConfigProvider {
         SearchRequest searchRequest = client.prepareSearch(MlConfigIndex.indexName())
                 .setIndicesOptions(IndicesOptions.lenientExpandOpen())
                 .setSource(sourceBuilder)
-                .setSize(AnomalyDetectorsIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW)
+                .setSize(MlConfigIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW)
                 .request();
 
         ExpandedIdsMatcher requiredMatches = new ExpandedIdsMatcher(tokens, allowNoMatch);

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java

@@ -66,7 +66,6 @@ import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig;
 import org.elasticsearch.xpack.core.ml.job.config.Detector;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
 import org.elasticsearch.xpack.core.ml.job.config.JobUpdate;
-import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.core.ml.utils.MlStrings;
 import org.elasticsearch.xpack.core.ml.utils.ToXContentParams;
@@ -93,7 +92,7 @@ import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
  * anomaly detector job configuration document
  *
  * The number of jobs returned in a search it limited to
- * {@link AnomalyDetectorsIndex#CONFIG_INDEX_MAX_RESULTS_WINDOW}.
+ * {@link MlConfigIndex#CONFIG_INDEX_MAX_RESULTS_WINDOW}.
  * In most cases we expect 10s or 100s of jobs to be defined and
  * a search for all jobs should return all.
  */
@@ -522,7 +521,7 @@ public class JobConfigProvider {
         SearchRequest searchRequest = client.prepareSearch(MlConfigIndex.indexName())
                 .setIndicesOptions(IndicesOptions.lenientExpandOpen())
                 .setSource(sourceBuilder)
-                .setSize(AnomalyDetectorsIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW)
+                .setSize(MlConfigIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW)
                 .request();
 
         ExpandedIdsMatcher requiredMatches = new ExpandedIdsMatcher(tokens, allowNoMatch);
@@ -580,7 +579,7 @@ public class JobConfigProvider {
         SearchRequest searchRequest = client.prepareSearch(MlConfigIndex.indexName())
                 .setIndicesOptions(IndicesOptions.lenientExpandOpen())
                 .setSource(sourceBuilder)
-                .setSize(AnomalyDetectorsIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW)
+                .setSize(MlConfigIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW)
                 .request();
 
         ExpandedIdsMatcher requiredMatches = new ExpandedIdsMatcher(tokens, allowNoMatch);
@@ -638,7 +637,7 @@ public class JobConfigProvider {
         SearchRequest searchRequest = client.prepareSearch(MlConfigIndex.indexName())
                 .setIndicesOptions(IndicesOptions.lenientExpandOpen())
                 .setSource(sourceBuilder)
-                .setSize(AnomalyDetectorsIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW)
+                .setSize(MlConfigIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW)
                 .request();
 
         executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, searchRequest,
@@ -700,7 +699,7 @@ public class JobConfigProvider {
         SearchRequest searchRequest = client.prepareSearch(MlConfigIndex.indexName())
                 .setIndicesOptions(IndicesOptions.lenientExpandOpen())
                 .setSource(sourceBuilder)
-                .setSize(AnomalyDetectorsIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW)
+                .setSize(MlConfigIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW)
                 .request();
 
         executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, searchRequest,

x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistryTests.java

@@ -84,7 +84,7 @@ public class MlIndexTemplateRegistryTests extends ESTestCase {
 
         registry.clusterChanged(createClusterChangedEvent(nodes));
 
-        verify(client.admin().indices(), times(7)).putTemplate(putIndexTemplateRequestCaptor.capture(), anyObject());
+        verify(client.admin().indices(), times(4)).putTemplate(putIndexTemplateRequestCaptor.capture(), anyObject());
 
         PutIndexTemplateRequest req = putIndexTemplateRequestCaptor.getAllValues().stream()
             .filter(r -> r.name().equals(AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX))
@@ -100,7 +100,7 @@ public class MlIndexTemplateRegistryTests extends ESTestCase {
 
         registry.clusterChanged(createClusterChangedEvent(nodes));
 
-        verify(client.admin().indices(), times(7)).putTemplate(putIndexTemplateRequestCaptor.capture(), anyObject());
+        verify(client.admin().indices(), times(4)).putTemplate(putIndexTemplateRequestCaptor.capture(), anyObject());
 
         PutIndexTemplateRequest req = putIndexTemplateRequestCaptor.getAllValues().stream()
             .filter(r -> r.name().equals(MlStatsIndex.TEMPLATE_NAME))

x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsActionTests.java

@@ -24,7 +24,6 @@ import org.elasticsearch.persistent.PersistentTasksCustomMetadata.Assignment;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
 import org.elasticsearch.xpack.core.ml.action.StartDataFrameAnalyticsAction.TaskParams;
-import org.elasticsearch.xpack.core.template.IndexTemplateConfig;
 import org.elasticsearch.xpack.ml.MachineLearning;
 import org.elasticsearch.xpack.ml.action.TransportStartDataFrameAnalyticsAction.TaskExecutor;
 import org.elasticsearch.xpack.ml.dataframe.DataFrameAnalyticsManager;
@@ -166,8 +165,7 @@ public class TransportStartDataFrameAnalyticsActionTests extends ESTestCase {
             mock(DataFrameAnalyticsManager.class),
             mock(DataFrameAnalyticsAuditor.class),
             mock(MlMemoryTracker.class),
-            new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)),
-            mock(IndexTemplateConfig.class));
+            new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)));
     }
 
     private static DiscoveryNode createNode(int i, boolean isMlNode, Version nodeVersion) {

x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java

@@ -19,6 +19,7 @@ import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.analysis.common.CommonAnalysisPlugin;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.client.OriginSettingClient;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
@@ -41,6 +42,7 @@ import org.elasticsearch.script.ScriptEngine;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.MockHttpTransport;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.action.util.QueryPage;
 import org.elasticsearch.xpack.core.ilm.LifecycleSettings;
@@ -469,7 +471,9 @@ public abstract class BaseMlIntegTestCase extends ESIntegTestCase {
      * Sets delayed allocation to 0 to make sure we have tests are not delayed
       */
     protected void setMlIndicesDelayedNodeLeftTimeoutToZero() {
-        client().admin().indices().updateSettings(new UpdateSettingsRequest(".ml-*")
+        OriginSettingClient originSettingClient = new OriginSettingClient(client(), ClientHelper.ML_ORIGIN);
+        originSettingClient.admin().indices().updateSettings(new UpdateSettingsRequest(".ml-*")
+            .origin(ClientHelper.ML_ORIGIN)
             .settings(Settings.builder().put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), 0).build()))
             .actionGet();
     }

x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/AbstractXPackRestTest.java

@@ -92,10 +92,8 @@ public class AbstractXPackRestTest extends ESClientYamlSuiteTestCase {
             templates.addAll(
                 Arrays.asList(
                     NotificationsIndex.NOTIFICATIONS_INDEX,
-                    MlMetaIndex.indexName(),
                     AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX,
                     AnomalyDetectorsIndex.jobResultsIndexPrefix(),
-                    MlConfigIndex.indexName(),
                     TransformInternalIndexConstants.AUDIT_INDEX,
                     TransformInternalIndexConstants.LATEST_INDEX_NAME
                 ));

x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java

@@ -45,7 +45,9 @@ public class MlConfigIndexMappingsFullClusterRestartIT extends AbstractFullClust
 
     @Before
     public void waitForMlTemplates() throws Exception {
-        List<String> templatesToWaitFor = XPackRestTestConstants.ML_POST_V660_TEMPLATES;
+        List<String> templatesToWaitFor = (isRunningAgainstOldCluster() && getOldClusterVersion().before(Version.V_7_12_0))
+            ? XPackRestTestConstants.ML_POST_V660_TEMPLATES
+            : XPackRestTestConstants.ML_POST_V7120_TEMPLATES;
         XPackRestTestHelper.waitForTemplates(client(), templatesToWaitFor);
     }
 

x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlMigrationFullClusterRestartIT.java

@@ -6,6 +6,7 @@
  */
 package org.elasticsearch.xpack.restart;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.common.Strings;
@@ -56,7 +57,9 @@ public class MlMigrationFullClusterRestartIT extends AbstractFullClusterRestartT
 
     @Before
     public void waitForMlTemplates() throws Exception {
-        List<String> templatesToWaitFor = XPackRestTestConstants.ML_POST_V660_TEMPLATES;
+        List<String> templatesToWaitFor = (isRunningAgainstOldCluster() && getOldClusterVersion().before(Version.V_7_12_0))
+            ? XPackRestTestConstants.ML_POST_V660_TEMPLATES
+            : XPackRestTestConstants.ML_POST_V7120_TEMPLATES;
         XPackRestTestHelper.waitForTemplates(client(), templatesToWaitFor);
     }
 

x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlJobSnapshotUpgradeIT.java

@@ -75,7 +75,10 @@ public class MlJobSnapshotUpgradeIT extends AbstractUpgradeTestCase {
 
     @Override
     protected Collection<String> templatesToWaitFor() {
-        return Stream.concat(XPackRestTestConstants.ML_POST_V660_TEMPLATES.stream(),
+        List<String> templatesToWaitFor = UPGRADE_FROM_VERSION.onOrAfter(Version.V_7_12_0)
+            ? XPackRestTestConstants.ML_POST_V7120_TEMPLATES
+            : XPackRestTestConstants.ML_POST_V660_TEMPLATES;
+        return Stream.concat(templatesToWaitFor.stream(),
             super.templatesToWaitFor().stream()).collect(Collectors.toSet());
     }
 

x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java

@@ -22,6 +22,7 @@ import org.elasticsearch.xpack.test.rest.XPackRestTestHelper;
 import java.io.IOException;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -34,7 +35,10 @@ public class MlMappingsUpgradeIT extends AbstractUpgradeTestCase {
 
     @Override
     protected Collection<String> templatesToWaitFor() {
-        return Stream.concat(XPackRestTestConstants.ML_POST_V660_TEMPLATES.stream(),
+        List<String> templatesToWaitFor = UPGRADE_FROM_VERSION.onOrAfter(Version.V_7_12_0)
+            ? XPackRestTestConstants.ML_POST_V7120_TEMPLATES
+            : XPackRestTestConstants.ML_POST_V660_TEMPLATES;
+        return Stream.concat(templatesToWaitFor.stream(),
             super.templatesToWaitFor().stream()).collect(Collectors.toSet());
     }
 

x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java

@@ -38,7 +38,7 @@ public class UpgradeClusterClientYamlTestSuiteIT extends ESClientYamlSuiteTestCa
     @Before
     public void waitForTemplates() throws Exception {
         try {
-            XPackRestTestHelper.waitForTemplates(client(), XPackRestTestConstants.ML_POST_V660_TEMPLATES);
+            XPackRestTestHelper.waitForTemplates(client(), XPackRestTestConstants.ML_POST_V7120_TEMPLATES);
         } catch (AssertionError e) {
             throw new AssertionError("Failure in test setup: Failed to initialize ML index templates", e);
         }

x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/IndexMappingTemplateAsserter.java

@@ -55,25 +55,6 @@ public class IndexMappingTemplateAsserter {
      * @throws IOException On error
      */
     public static void assertMlMappingsMatchTemplates(RestClient client) throws IOException {
-        // Keys that have been dynamically mapped in the .ml-config index
-        // but are not in the template. These can only be fixed with
-        // re-index and should be addressed at the next major upgrade.
-        // For now this serves as documentation of the missing fields
-        Set<String> configIndexExceptions = new HashSet<>();
-        configIndexExceptions.add("properties.allow_lazy_start.type");
-        configIndexExceptions.add("properties.analysis.properties.classification.properties.randomize_seed.type");
-        configIndexExceptions.add("properties.analysis.properties.outlier_detection.properties.compute_feature_influence.type");
-        configIndexExceptions.add("properties.analysis.properties.outlier_detection.properties.outlier_fraction.type");
-        configIndexExceptions.add("properties.analysis.properties.outlier_detection.properties.standardization_enabled.type");
-        configIndexExceptions.add("properties.analysis.properties.regression.properties.randomize_seed.type");
-        configIndexExceptions.add("properties.deleting.type");
-        configIndexExceptions.add("properties.model_memory_limit.type");
-
-        // renamed to max_trees in 7.7.
-        // These exceptions are necessary for Full Cluster Restart tests where the upgrade version is < 7.x
-        configIndexExceptions.add("properties.analysis.properties.classification.properties.maximum_number_trees.type");
-        configIndexExceptions.add("properties.analysis.properties.regression.properties.maximum_number_trees.type");
-
         // Excluding those from stats index as some have been renamed and other removed.
         // These exceptions are necessary for Full Cluster Restart tests where the upgrade version is < 7.x
         Set<String> statsIndexException = new HashSet<>();
@@ -88,18 +69,21 @@ public class IndexMappingTemplateAsserter {
         Set<String> notificationsIndexExceptions = new HashSet<>();
         notificationsIndexExceptions.add("properties.message.fields.raw.ignore_above");
 
-        assertLegacyTemplateMatchesIndexMappings(client, ".ml-config", ".ml-config", false, configIndexExceptions, true);
-        // the true parameter means the index may not have been created
-        assertLegacyTemplateMatchesIndexMappings(client, ".ml-meta", ".ml-meta", true, Collections.emptySet(), true);
         assertLegacyTemplateMatchesIndexMappings(client, ".ml-stats", ".ml-stats-000001", true, statsIndexException, false);
         assertLegacyTemplateMatchesIndexMappings(client, ".ml-state", ".ml-state-000001", true, Collections.emptySet(), false);
         // Depending on the order Full Cluster restart tests are run there may not be an notifications index yet
         assertLegacyTemplateMatchesIndexMappings(client,
             ".ml-notifications-000001", ".ml-notifications-000001", true, notificationsIndexExceptions, false);
-        assertLegacyTemplateMatchesIndexMappings(client,
-            ".ml-inference-000003", ".ml-inference-000003", true, Collections.emptySet(), true);
         // .ml-annotations-6 does not use a template
         // .ml-anomalies-shared uses a template but will have dynamically updated mappings as new jobs are opened
+
+        // Dynamic mappings updates are banned for system indices.
+        // The .ml-config and .ml-meta indices have mappings that allow dynamic updates.
+        // The effect is instant error if a document containing an unknown field is added
+        // to one of these indices.  Assuming we have some sort of test coverage somewhere
+        // for new fields, we will very quickly catch any failures to add new fields to
+        // the mappings for the .ml-config and .ml-meta indices.  So there is no need to
+        // test again here.
     }
 
     /**

x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/XPackRestTestConstants.java

@@ -34,6 +34,11 @@ public final class XPackRestTestConstants {
             RESULTS_INDEX_PREFIX,
             CONFIG_INDEX);
 
+    public static final List<String> ML_POST_V7120_TEMPLATES =
+        List.of(
+            STATE_INDEX_PREFIX,
+            RESULTS_INDEX_PREFIX);
+
     // Transform constants:
     public static final String TRANSFORM_TASK_NAME = "data_frame/transforms";
     public static final String TRANSFORM_INTERNAL_INDEX_PREFIX = ".transform-internal-";