Browse Source

[ML] DFA jobs should accept excluding an unsupported field (#49535)

Before this change excluding an unsupported field resulted in
an error message that explained the excluded field could not be
detected as if it doesn't exist. This error message is confusing.

This commit commit changes this so that there is no error in this
scenario. When excluding a field that does exist but has been
automatically been excluded from the analysis there is no harm
(unlike excluding a missing field which could be a typo).
Dimitris Athanasiou 5 years ago
parent
commit
0215653c01

+ 1 - 1
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java

@@ -203,7 +203,7 @@ public class ExtractedFieldsDetector {
                 .expand(includes, false);
                 .expand(includes, false);
             // If the exclusion set does not match anything, that means the fields are already not present
             // If the exclusion set does not match anything, that means the fields are already not present
             // no need to raise if nothing matched
             // no need to raise if nothing matched
-            Set<String> excludedSet = NameResolver.newUnaliased(fields,
+            Set<String> excludedSet = NameResolver.newUnaliased(fieldCapabilitiesResponse.get().keySet(),
                 (ex) -> new ResourceNotFoundException(
                 (ex) -> new ResourceNotFoundException(
                     Messages.getMessage(Messages.DATA_FRAME_ANALYTICS_BAD_FIELD_FILTER, ex)))
                     Messages.getMessage(Messages.DATA_FRAME_ANALYTICS_BAD_FIELD_FILTER, ex)))
                 .expand(excludes, true);
                 .expand(excludes, true);

+ 36 - 0
x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java

@@ -300,6 +300,42 @@ public class ExtractedFieldsDetectorTests extends ESTestCase {
         assertThat(e.getMessage(), equalTo("No field [_id] could be detected"));
         assertThat(e.getMessage(), equalTo("No field [_id] could be detected"));
     }
     }
 
 
+    public void testDetect_GivenExcludedFieldIsMissing() {
+        FieldCapabilitiesResponse fieldCapabilities = new MockFieldCapsResponseBuilder()
+            .addAggregatableField("foo", "float")
+            .build();
+        FetchSourceContext analyzedFields = new FetchSourceContext(true, new String[]{"*"}, new String[] {"bar"});
+
+        ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector(
+            SOURCE_INDEX, buildOutlierDetectionConfig(analyzedFields), false, 100, fieldCapabilities, Collections.emptyMap());
+        ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> extractedFieldsDetector.detect());
+
+        assertThat(e.getMessage(), equalTo("No field [bar] could be detected"));
+    }
+
+    public void testDetect_GivenExcludedFieldIsUnsupported() {
+        FieldCapabilitiesResponse fieldCapabilities = new MockFieldCapsResponseBuilder()
+            .addAggregatableField("numeric", "float")
+            .addAggregatableField("categorical", "keyword")
+            .build();
+        FetchSourceContext analyzedFields = new FetchSourceContext(true, null, new String[] {"categorical"});
+
+        ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector(
+            SOURCE_INDEX, buildOutlierDetectionConfig(analyzedFields), false, 100, fieldCapabilities, Collections.emptyMap());
+
+        Tuple<ExtractedFields, List<FieldSelection>> fieldExtraction = extractedFieldsDetector.detect();
+
+        List<ExtractedField> allFields = fieldExtraction.v1().getAllFields();
+        assertThat(allFields.size(), equalTo(1));
+        assertThat(allFields.get(0).getName(), equalTo("numeric"));
+
+        assertFieldSelectionContains(fieldExtraction.v2(),
+            FieldSelection.excluded("categorical", Collections.singleton("keyword"),
+                "unsupported type; supported types are [boolean, byte, double, float, half_float, integer, long, scaled_float, short]"),
+            FieldSelection.included("numeric", Collections.singleton("float"), false, FieldSelection.FeatureType.NUMERICAL)
+        );
+    }
+
     public void testDetect_ShouldSortFieldsAlphabetically() {
     public void testDetect_ShouldSortFieldsAlphabetically() {
         int fieldCount = randomIntBetween(10, 20);
         int fieldCount = randomIntBetween(10, 20);
         List<String> fields = new ArrayList<>();
         List<String> fields = new ArrayList<>();