[ML] renaming inference processor field field_mappings to new name field_map (#53433)

This renames the `inference` processor configuration field `field_mappings` to `field_map`. 

`field_mappings` is now deprecated.

client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java

@@ -2265,7 +2265,7 @@ public class MachineLearningIT extends ESRestHighLevelClientTestCase {
             "          \"target_field\": \"regression_value\",\n" +
             "          \"model_id\": \"" + modelIdPrefix + 0 + "\",\n" +
             "          \"inference_config\": {\"regression\": {}},\n" +
-            "          \"field_mappings\": {\n" +
+            "          \"field_map\": {\n" +
             "            \"col1\": \"col1\",\n" +
             "            \"col2\": \"col2\",\n" +
             "            \"col3\": \"col3\",\n" +

docs/reference/ingest/processors/inference.asciidoc

@@ -14,7 +14,7 @@ ingested in the pipeline.
 | Name               | Required  | Default                        | Description
 | `model_id`         | yes       | -                              | (String) The ID of the model to load and infer against.
 | `target_field`     | no        | `ml.inference.<processor_tag>` | (String) Field added to incoming documents to contain results objects.
-| `field_mappings`   | yes       | -                              | (Object) Maps the document field names to the known field names of the model. This mapping takes precedence over any default mappings provided in the model configuration.
+| `field_map`        | yes       | -                              | (Object) Maps the document field names to the known field names of the model. This mapping takes precedence over any default mappings provided in the model configuration.
 | `inference_config` | yes       | -                              | (Object) Contains the inference type and its options. There are two types: <<inference-processor-regression-opt,`regression`>> and <<inference-processor-classification-opt,`classification`>>.
 include::common-options.asciidoc[]
 |======
@@ -26,7 +26,7 @@ include::common-options.asciidoc[]
   "inference": {
     "model_id": "flight_delay_regression-1571767128603",
     "target_field": "FlightDelayMin_prediction_infer",
-    "field_mappings": {},
+    "field_map": {},
     "inference_config": { "regression": {} }
   }
 }

x-pack/plugin/ml/qa/ml-with-security/build.gradle

@@ -139,6 +139,9 @@ integTest.runner {
     'ml/inference_crud/Test put ensemble with empty models',
     'ml/inference_crud/Test put ensemble with tree where tree has out of bounds feature_names index',
     'ml/inference_crud/Test put model with empty input.field_names',
+    'ml/inference_processor/Test create processor with missing mandatory fields',
+    'ml/inference_processor/Test create and delete pipeline with inference processor',
+    'ml/inference_processor/Test create processor with deprecated fields',
     'ml/inference_stats_crud/Test get stats given missing trained model',
     'ml/inference_stats_crud/Test get stats given expression without matches and allow_no_match is false',
     'ml/jobs_crud/Test cannot create job with existing categorizer state document',

x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/InferenceIngestIT.java

@@ -120,7 +120,7 @@ public class InferenceIngestIT extends ESRestTestCase {
             "                \"num_top_feature_importance_values\": 2" +
             "          }},\n" +
             "          \"model_id\": \"test_classification\",\n" +
-            "          \"field_mappings\": {\n" +
+            "          \"field_map\": {\n" +
             "            \"col1\": \"col1\",\n" +
             "            \"col2\": \"col2\",\n" +
             "            \"col3\": \"col3\",\n" +
@@ -133,7 +133,7 @@ public class InferenceIngestIT extends ESRestTestCase {
             "          \"target_field\": \"ml.regression\",\n" +
             "          \"model_id\": \"test_regression\",\n" +
             "          \"inference_config\": {\"regression\":{}},\n" +
-            "          \"field_mappings\": {\n" +
+            "          \"field_map\": {\n" +
             "            \"col1\": \"col1\",\n" +
             "            \"col2\": \"col2\",\n" +
             "            \"col3\": \"col3\",\n" +
@@ -166,7 +166,7 @@ public class InferenceIngestIT extends ESRestTestCase {
             "        \"inference\": {\n" +
             "          \"model_id\": \"test_classification_missing\",\n" +
             "          \"inference_config\": {\"classification\":{}},\n" +
-            "          \"field_mappings\": {\n" +
+            "          \"field_map\": {\n" +
             "            \"col1\": \"col1\",\n" +
             "            \"col2\": \"col2\",\n" +
             "            \"col3\": \"col3\",\n" +
@@ -204,7 +204,7 @@ public class InferenceIngestIT extends ESRestTestCase {
             "                \"num_top_feature_importance_values\": 2" +
             "          }},\n" +
             "          \"model_id\": \"test_classification\",\n" +
-            "          \"field_mappings\": {}\n" +
+            "          \"field_map\": {}\n" +
             "        }\n" +
             "      }\n"+
             "    ]\n" +
@@ -233,7 +233,7 @@ public class InferenceIngestIT extends ESRestTestCase {
             "        \"inference\": {\n" +
             "          \"inference_config\": {\"classification\":{}},\n" +
             "          \"model_id\": \"lang_ident_model_1\",\n" +
-            "          \"field_mappings\": {}\n" +
+            "          \"field_map\": {}\n" +
             "        }\n" +
             "      }\n" +
             "    ]\n" +
@@ -570,7 +570,7 @@ public class InferenceIngestIT extends ESRestTestCase {
         "          \"model_id\": \"test_classification\",\n" +
         "          \"tag\": \"classification\",\n" +
         "          \"inference_config\": {\"classification\": {}},\n" +
-        "          \"field_mappings\": {\n" +
+        "          \"field_map\": {\n" +
         "            \"col1\": \"col1\",\n" +
         "            \"col2\": \"col2\",\n" +
         "            \"col3\": \"col3\",\n" +
@@ -586,7 +586,7 @@ public class InferenceIngestIT extends ESRestTestCase {
         "          \"model_id\": \"test_regression\",\n" +
         "          \"tag\": \"regression\",\n" +
         "          \"inference_config\": {\"regression\": {}},\n" +
-        "          \"field_mappings\": {\n" +
+        "          \"field_map\": {\n" +
         "            \"col1\": \"col1\",\n" +
         "            \"col2\": \"col2\",\n" +
         "            \"col3\": \"col3\",\n" +

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java

@@ -18,6 +18,7 @@ import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
@@ -63,6 +64,7 @@ public class InferenceProcessor extends AbstractProcessor {
     public static final String INFERENCE_CONFIG = "inference_config";
     public static final String TARGET_FIELD = "target_field";
     public static final String FIELD_MAPPINGS = "field_mappings";
+    public static final String FIELD_MAP = "field_map";
     private static final String DEFAULT_TARGET_FIELD = "ml.inference";
 
     private final Client client;
@@ -70,7 +72,7 @@ public class InferenceProcessor extends AbstractProcessor {
 
     private final String targetField;
     private final InferenceConfig inferenceConfig;
-    private final Map<String, String> fieldMapping;
+    private final Map<String, String> fieldMap;
     private final InferenceAuditor auditor;
     private volatile boolean previouslyLicensed;
     private final AtomicBoolean shouldAudit = new AtomicBoolean(true);
@@ -81,14 +83,14 @@ public class InferenceProcessor extends AbstractProcessor {
                               String targetField,
                               String modelId,
                               InferenceConfig inferenceConfig,
-                              Map<String, String> fieldMapping) {
+                              Map<String, String> fieldMap) {
         super(tag);
         this.client = ExceptionsHelper.requireNonNull(client, "client");
         this.targetField = ExceptionsHelper.requireNonNull(targetField, TARGET_FIELD);
         this.auditor = ExceptionsHelper.requireNonNull(auditor, "auditor");
         this.modelId = ExceptionsHelper.requireNonNull(modelId, MODEL_ID);
         this.inferenceConfig = ExceptionsHelper.requireNonNull(inferenceConfig, INFERENCE_CONFIG);
-        this.fieldMapping = ExceptionsHelper.requireNonNull(fieldMapping, FIELD_MAPPINGS);
+        this.fieldMap = ExceptionsHelper.requireNonNull(fieldMap, FIELD_MAP);
     }
 
     public String getModelId() {
@@ -126,7 +128,7 @@ public class InferenceProcessor extends AbstractProcessor {
 
     InternalInferModelAction.Request buildRequest(IngestDocument ingestDocument) {
         Map<String, Object> fields = new HashMap<>(ingestDocument.getSourceAndMetadata());
-        Model.mapFieldsIfNecessary(fields, fieldMapping);
+        Model.mapFieldsIfNecessary(fields, fieldMap);
         return new InternalInferModelAction.Request(modelId, fields, inferenceConfig, previouslyLicensed);
     }
 
@@ -235,7 +237,14 @@ public class InferenceProcessor extends AbstractProcessor {
             // If multiple inference processors are in the same pipeline, it is wise to tag them
             // The tag will keep default value entries from stepping on each other
             String targetField = ConfigurationUtils.readStringProperty(TYPE, tag, config, TARGET_FIELD, defaultTargetField);
-            Map<String, String> fieldMapping = ConfigurationUtils.readOptionalMap(TYPE, tag, config, FIELD_MAPPINGS);
+            Map<String, String> fieldMap = ConfigurationUtils.readOptionalMap(TYPE, tag, config, FIELD_MAP);
+            if (fieldMap == null) {
+                fieldMap = ConfigurationUtils.readOptionalMap(TYPE, tag, config, FIELD_MAPPINGS);
+                //TODO Remove in 8.x
+                if (fieldMap != null) {
+                    LoggingDeprecationHandler.INSTANCE.usedDeprecatedName(FIELD_MAPPINGS, FIELD_MAP);
+                }
+            }
             InferenceConfig inferenceConfig = inferenceConfigFromMap(ConfigurationUtils.readMap(TYPE, tag, config, INFERENCE_CONFIG));
 
             return new InferenceProcessor(client,
@@ -244,7 +253,7 @@ public class InferenceProcessor extends AbstractProcessor {
                 targetField,
                 modelId,
                 inferenceConfig,
-                fieldMapping);
+                fieldMap);
         }
 
         // Package private for testing

x-pack/plugin/ml/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java

@@ -484,7 +484,7 @@ public class MachineLearningLicensingTests extends BaseMlIntegTestCase {
             "          \"target_field\": \"regression_value\",\n" +
             "          \"model_id\": \"modelprocessorlicensetest\",\n" +
             "          \"inference_config\": {\"regression\": {}},\n" +
-            "          \"field_mappings\": {}\n" +
+            "          \"field_map\": {}\n" +
             "        }\n" +
             "      }]}\n";
         // Creating a pipeline should work

x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsActionTests.java

@@ -267,7 +267,7 @@ public class TransportGetTrainedModelsStatsActionTests extends ESTestCase {
                     new HashMap<String, Object>() {{
                         put(InferenceProcessor.MODEL_ID, modelId);
                         put("inference_config", Collections.singletonMap("regression", Collections.emptyMap()));
-                        put("field_mappings", Collections.emptyMap());
+                        put("field_map", Collections.emptyMap());
                         put("target_field", randomAlphaOfLength(10));
                     }}))))) {
             return new PipelineConfiguration("pipeline_with_model_" + modelId + num,

x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessorFactoryTests.java

@@ -131,7 +131,7 @@ public class InferenceProcessorFactoryTests extends ESTestCase {
             ingestService);
 
         Map<String, Object> config = new HashMap<>() {{
-            put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
+            put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
             put(InferenceProcessor.MODEL_ID, "my_model");
             put(InferenceProcessor.TARGET_FIELD, "result");
             put(InferenceProcessor.INFERENCE_CONFIG, Collections.singletonMap("unknown_type", Collections.emptyMap()));
@@ -143,7 +143,7 @@ public class InferenceProcessorFactoryTests extends ESTestCase {
             equalTo("unrecognized inference configuration type [unknown_type]. Supported types [classification, regression]"));
 
         Map<String, Object> config2 = new HashMap<>() {{
-            put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
+            put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
             put(InferenceProcessor.MODEL_ID, "my_model");
             put(InferenceProcessor.TARGET_FIELD, "result");
             put(InferenceProcessor.INFERENCE_CONFIG, Collections.singletonMap("regression", "boom"));
@@ -154,7 +154,7 @@ public class InferenceProcessorFactoryTests extends ESTestCase {
             equalTo("inference_config must be an object with one inference type mapped to an object."));
 
         Map<String, Object> config3 = new HashMap<>() {{
-            put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
+            put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
             put(InferenceProcessor.MODEL_ID, "my_model");
             put(InferenceProcessor.TARGET_FIELD, "result");
             put(InferenceProcessor.INFERENCE_CONFIG, Collections.emptyMap());
@@ -173,7 +173,7 @@ public class InferenceProcessorFactoryTests extends ESTestCase {
         processorFactory.accept(builderClusterStateWithModelReferences(Version.V_7_5_0, "model1"));
 
         Map<String, Object> regression = new HashMap<>() {{
-            put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
+            put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
             put(InferenceProcessor.MODEL_ID, "my_model");
             put(InferenceProcessor.TARGET_FIELD, "result");
             put(InferenceProcessor.INFERENCE_CONFIG,
@@ -191,7 +191,7 @@ public class InferenceProcessorFactoryTests extends ESTestCase {
         }
 
         Map<String, Object> classification = new HashMap<>() {{
-            put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
+            put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
             put(InferenceProcessor.MODEL_ID, "my_model");
             put(InferenceProcessor.TARGET_FIELD, "result");
             put(InferenceProcessor.INFERENCE_CONFIG, Collections.singletonMap(ClassificationConfig.NAME.getPreferredName(),
@@ -216,7 +216,7 @@ public class InferenceProcessorFactoryTests extends ESTestCase {
             ingestService);
 
         Map<String, Object> regression = new HashMap<>() {{
-            put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
+            put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
             put(InferenceProcessor.MODEL_ID, "my_model");
             put(InferenceProcessor.TARGET_FIELD, "result");
             put(InferenceProcessor.INFERENCE_CONFIG,
@@ -230,7 +230,7 @@ public class InferenceProcessorFactoryTests extends ESTestCase {
         }
 
         Map<String, Object> classification = new HashMap<>() {{
-            put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
+            put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
             put(InferenceProcessor.MODEL_ID, "my_model");
             put(InferenceProcessor.TARGET_FIELD, "result");
             put(InferenceProcessor.INFERENCE_CONFIG, Collections.singletonMap(ClassificationConfig.NAME.getPreferredName(),
@@ -251,7 +251,7 @@ public class InferenceProcessorFactoryTests extends ESTestCase {
             ingestService);
 
         Map<String, Object> regression = new HashMap<>() {{
-            put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
+            put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
             put(InferenceProcessor.MODEL_ID, "my_model");
             put(InferenceProcessor.TARGET_FIELD, "ml");
             put(InferenceProcessor.INFERENCE_CONFIG, Collections.singletonMap(RegressionConfig.NAME.getPreferredName(),
@@ -305,7 +305,7 @@ public class InferenceProcessorFactoryTests extends ESTestCase {
                         put(InferenceProcessor.INFERENCE_CONFIG,
                                 Collections.singletonMap(RegressionConfig.NAME.getPreferredName(), Collections.emptyMap()));
                         put(InferenceProcessor.TARGET_FIELD, "new_field");
-                        put(InferenceProcessor.FIELD_MAPPINGS, Collections.singletonMap("source", "dest"));
+                        put(InferenceProcessor.FIELD_MAP, Collections.singletonMap("source", "dest"));
                     }}))))) {
             return new PipelineConfiguration("pipeline_with_model_" + modelId, BytesReference.bytes(xContentBuilder), XContentType.JSON);
         }

x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_crud.yml

@@ -313,7 +313,7 @@ setup:
                   "model_id" : "a-regression-model-0",
                   "inference_config": {"regression": {}},
                   "target_field": "regression_field",
-                  "field_mappings": {}
+                  "field_map": {}
                 }
               }
             ]

x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_processor.yml

@@ -0,0 +1,114 @@
+# These tests are not part of the ml-with-security tests
+# These are more like ingest processor tests than actual ML focused tests.
+setup:
+  - skip:
+      features: headers
+  - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
+      ml.put_trained_model:
+        model_id: a-perfect-regression-model
+        body: >
+          {
+            "description": "empty model for tests",
+            "tags": ["regression", "tag1"],
+            "input": {"field_names": ["field1", "field2"]},
+            "definition": {
+               "preprocessors": [],
+               "trained_model": {
+                  "tree": {
+                     "feature_names": ["field1", "field2"],
+                     "tree_structure": [
+                        {"node_index": 0, "leaf_value": 42}
+                     ],
+                     "target_type": "regression"
+                  }
+               }
+            }
+          }
+
+---
+"Test create and delete pipeline with inference processor":
+  - do:
+      ingest.put_pipeline:
+        id: "regression-model-pipeline"
+        body:  >
+          {
+            "processors": [
+              {
+                "inference" : {
+                  "model_id" : "a-perfect-regression-model",
+                  "inference_config": {"regression": {}},
+                  "target_field": "regression_field",
+                  "field_map": {}
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+  - do:
+      ingest.delete_pipeline:
+        id: "regression-model-pipeline"
+---
+"Test create processor with missing mandatory fields":
+  - do:
+      catch: /\[model_id\] required property is missing/
+      ingest.put_pipeline:
+        id: "regression-model-pipeline"
+        body:  >
+          {
+            "processors": [
+              {
+                "inference" : {
+                  "inference_config": {"regression": {}},
+                  "target_field": "regression_field",
+                  "field_map": {}
+                }
+              }
+            ]
+          }
+  - do:
+      catch: /\[inference_config\] required property is missing/
+      ingest.put_pipeline:
+        id: "regression-model-pipeline"
+        body:  >
+          {
+            "processors": [
+              {
+                "inference" : {
+                  "model_id" : "a-perfect-regression-model",
+                  "target_field": "regression_field",
+                  "field_map": {}
+                }
+              }
+            ]
+          }
+---
+"Test create processor with deprecated fields":
+  - skip:
+      features:
+       - "warnings"
+       - "allowed_warnings"
+  - do:
+      warnings:
+        - 'Deprecated field [field_mappings] used, expected [field_map] instead'
+      ingest.put_pipeline:
+        id: "regression-model-pipeline"
+        body:  >
+          {
+            "processors": [
+              {
+                "inference" : {
+                  "model_id" : "a-perfect-regression-model",
+                  "inference_config": {"regression": {}},
+                  "field_mappings": {}
+                }
+              }
+            ]
+          }
+
+  - do:
+      allowed_warnings:
+        - 'Deprecated field [field_mappings] used, expected [field_map] instead'
+      ingest.delete_pipeline:
+        id: "regression-model-pipeline"

x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_stats_crud.yml

@@ -66,7 +66,7 @@ setup:
                   "model_id" : "a-used-regression-model",
                   "inference_config": {"regression": {}},
                   "target_field": "regression_field",
-                  "field_mappings": {}
+                  "field_map": {}
                 }
               }
             ]
@@ -84,7 +84,7 @@ setup:
                   "model_id" : "a-used-regression-model",
                   "inference_config": {"regression": {}},
                   "target_field": "regression_field",
-                  "field_mappings": {}
+                  "field_map": {}
                 }
               }
             ]