Option to disable device type parsing in user agent processor (#71625)

docs/reference/ingest/common-log-format-example.asciidoc

@@ -247,8 +247,7 @@ The API returns:
             },
             "name": "Chrome",
             "device": {
-              "name": "Mac",
-              "type": "Desktop"
+              "name": "Mac"
             },
             "version": "52.0.2743.116"
           }

docs/reference/ingest/processors/user-agent.asciidoc

@@ -69,8 +69,7 @@ Which returns
         "full": "Mac OS X 10.10.5"
       },
       "device" : {
-        "name" : "Mac",
-        "type" : "Desktop"
+        "name" : "Mac"
       },
     }
   }

modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java

@@ -153,14 +153,14 @@ final class UserAgentParser {
         return name;
     }
 
-    public Details parse(String agentString) {
+    public Details parse(String agentString, boolean extractDeviceType) {
         Details details = cache.get(name, agentString);
 
         if (details == null) {
             VersionedName userAgent = findMatch(uaPatterns, agentString);
             VersionedName operatingSystem = findMatch(osPatterns, agentString);
             VersionedName device = findMatch(devicePatterns, agentString);
-            String deviceType = deviceTypeParser.findDeviceType(agentString, userAgent, operatingSystem, device);
+            String deviceType = extractDeviceType ? deviceTypeParser.findDeviceType(agentString, userAgent, operatingSystem, device) : null;
             details = new Details(userAgent, operatingSystem, device, deviceType);
             cache.put(name, agentString, details);
         }

modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java

@@ -38,18 +38,24 @@ public class UserAgentProcessor extends AbstractProcessor {
     private final String targetField;
     private final Set<Property> properties;
     private final UserAgentParser parser;
+    private final boolean extractDeviceType;
     private final boolean ignoreMissing;
 
     public UserAgentProcessor(String tag, String description, String field, String targetField, UserAgentParser parser,
-                              Set<Property> properties, boolean ignoreMissing) {
+                              Set<Property> properties, boolean extractDeviceType, boolean ignoreMissing) {
         super(tag, description);
         this.field = field;
         this.targetField = targetField;
         this.parser = parser;
         this.properties = properties;
+        this.extractDeviceType = extractDeviceType;
         this.ignoreMissing = ignoreMissing;
     }
 
+    boolean isExtractDeviceType() {
+        return extractDeviceType;
+    }
+
     boolean isIgnoreMissing() {
         return ignoreMissing;
     }
@@ -64,7 +70,7 @@ public class UserAgentProcessor extends AbstractProcessor {
             throw new IllegalArgumentException("field [" + field + "] is null, cannot parse user-agent.");
         }
 
-        Details uaClient = parser.parse(userAgent);
+        Details uaClient = parser.parse(userAgent, extractDeviceType);
 
         Map<String, Object> uaDetails = new HashMap<>();
 
@@ -125,13 +131,17 @@ public class UserAgentProcessor extends AbstractProcessor {
                     Map<String, String> deviceDetails = new HashMap<>(1);
                     if (uaClient.device != null && uaClient.device.name != null) {
                         deviceDetails.put("name", uaClient.device.name);
-                        deviceDetails.put("type", uaClient.deviceType);
+                        if (extractDeviceType) {
+                            deviceDetails.put("type", uaClient.deviceType);
+                        }
                     } else {
                         deviceDetails.put("name", "Other");
-                        if (uaClient.deviceType != null) {
-                            deviceDetails.put("type", uaClient.deviceType);
-                        } else {
-                            deviceDetails.put("type", "Other");
+                        if (extractDeviceType) {
+                            if (uaClient.deviceType != null) {
+                                deviceDetails.put("type", uaClient.deviceType);
+                            } else {
+                                deviceDetails.put("type", "Other");
+                            }
                         }
                     }
                     uaDetails.put("device", deviceDetails);
@@ -179,6 +189,7 @@ public class UserAgentProcessor extends AbstractProcessor {
             String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "user_agent");
             String regexFilename = readStringProperty(TYPE, processorTag, config, "regex_file", IngestUserAgentPlugin.DEFAULT_PARSER_NAME);
             List<String> propertyNames = readOptionalList(TYPE, processorTag, config, "properties");
+            boolean extractDeviceType = readBooleanProperty(TYPE, processorTag, config, "extract_device_type", false);
             boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
             Object ecsValue = config.remove("ecs");
             if (ecsValue != null) {
@@ -206,7 +217,8 @@ public class UserAgentProcessor extends AbstractProcessor {
                 properties = EnumSet.allOf(Property.class);
             }
 
-            return new UserAgentProcessor(processorTag, description, field, targetField, parser, properties, ignoreMissing);
+            return new
+                UserAgentProcessor(processorTag, description, field, targetField, parser, properties, extractDeviceType, ignoreMissing);
         }
     }
 

modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java

@@ -78,6 +78,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
         assertThat(processor.getUaParser().getOsPatterns().size(), greaterThan(0));
         assertThat(processor.getUaParser().getDevicePatterns().size(), greaterThan(0));
         assertThat(processor.getProperties(), equalTo(EnumSet.allOf(UserAgentProcessor.Property.class)));
+        assertFalse(processor.isExtractDeviceType());
         assertFalse(processor.isIgnoreMissing());
     }
 
@@ -127,6 +128,19 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
         assertThat(processor.getUaParser().getDevicePatterns().size(), equalTo(0));
     }
 
+    public void testBuildExtractDeviceType() throws Exception {
+        UserAgentProcessor.Factory factory = new UserAgentProcessor.Factory(userAgentParsers);
+        boolean extractDeviceType = randomBoolean();
+
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "_field");
+        config.put("extract_device_type", extractDeviceType);
+
+        UserAgentProcessor processor = factory.create(null, null, null, config);
+        assertThat(processor.getField(), equalTo("_field"));
+        assertThat(processor.isExtractDeviceType(), equalTo(extractDeviceType));
+    }
+
     public void testBuildNonExistingRegexFile() throws Exception {
         UserAgentProcessor.Factory factory = new UserAgentProcessor.Factory(userAgentParsers);
 

modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java

@@ -40,12 +40,12 @@ public class UserAgentProcessorTests extends ESTestCase {
         UserAgentParser parser = new UserAgentParser(randomAlphaOfLength(10), regexStream, deviceTypeRegexStream, new UserAgentCache(1000));
 
         processor = new UserAgentProcessor(randomAlphaOfLength(10), null, "source_field", "target_field", parser,
-                EnumSet.allOf(UserAgentProcessor.Property.class), false);
+                EnumSet.allOf(UserAgentProcessor.Property.class), true, false);
     }
 
     public void testNullValueWithIgnoreMissing() throws Exception {
         UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), null, "source_field", "target_field", null,
-            EnumSet.allOf(UserAgentProcessor.Property.class), true);
+            EnumSet.allOf(UserAgentProcessor.Property.class), false, true);
         IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(),
             Collections.singletonMap("source_field", null));
         IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
@@ -55,7 +55,7 @@ public class UserAgentProcessorTests extends ESTestCase {
 
     public void testNonExistentWithIgnoreMissing() throws Exception {
         UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), null, "source_field", "target_field", null,
-            EnumSet.allOf(UserAgentProcessor.Property.class), true);
+            EnumSet.allOf(UserAgentProcessor.Property.class), false, true);
         IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
         IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
         processor.execute(ingestDocument);
@@ -64,7 +64,7 @@ public class UserAgentProcessorTests extends ESTestCase {
 
     public void testNullWithoutIgnoreMissing() throws Exception {
         UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), null, "source_field", "target_field", null,
-            EnumSet.allOf(UserAgentProcessor.Property.class), false);
+            EnumSet.allOf(UserAgentProcessor.Property.class), false, false);
         IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(),
             Collections.singletonMap("source_field", null));
         IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
@@ -74,7 +74,7 @@ public class UserAgentProcessorTests extends ESTestCase {
 
     public void testNonExistentWithoutIgnoreMissing() throws Exception {
         UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), null, "source_field", "target_field", null,
-            EnumSet.allOf(UserAgentProcessor.Property.class), false);
+            EnumSet.allOf(UserAgentProcessor.Property.class), false, false);
         IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
         IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
         Exception exception = expectThrows(Exception.class, () -> processor.execute(ingestDocument));
@@ -243,4 +243,34 @@ public class UserAgentProcessorTests extends ESTestCase {
         device.put("type", "Other");
         assertThat(target.get("device"), is(device));
     }
+
+    @SuppressWarnings("unchecked")
+    public void testExtractDeviceTypeDisabled() {
+        Map<String, Object> document = new HashMap<>();
+        document.put("source_field",
+            "Something I made up v42.0.1");
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+
+        InputStream regexStream = UserAgentProcessor.class.getResourceAsStream("/regexes.yml");
+        InputStream deviceTypeRegexStream = UserAgentProcessor.class.getResourceAsStream("/device_type_regexes.yml");
+        UserAgentParser parser = new UserAgentParser(randomAlphaOfLength(10), regexStream, deviceTypeRegexStream, new UserAgentCache(1000));
+        UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), null, "source_field", "target_field", parser,
+            EnumSet.allOf(UserAgentProcessor.Property.class), false, false);
+        processor.execute(ingestDocument);
+        Map<String, Object> data = ingestDocument.getSourceAndMetadata();
+
+        assertThat(data, hasKey("target_field"));
+        Map<String, Object> target = (Map<String, Object>) data.get("target_field");
+
+        assertThat(target.get("name"), is("Other"));
+        assertNull(target.get("major"));
+        assertNull(target.get("minor"));
+        assertNull(target.get("patch"));
+        assertNull(target.get("build"));
+
+        assertNull(target.get("os"));
+        Map<String, String> device = new HashMap<>();
+        device.put("name", "Other");
+        assertThat(target.get("device"), is(device));
+    }
 }

modules/ingest-user-agent/src/yamlRestTest/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml

@@ -32,7 +32,6 @@
   - match: { _source.user_agent.original: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" }
   - match: { _source.user_agent.os: {"name":"Mac OS X", "version":"10.9.2", "full":"Mac OS X 10.9.2"} }
   - match: { _source.user_agent.version: "33.0.1750.149" }
-  - match: { _source.user_agent.device: {"name": "Mac", type: "Desktop" }}
 
 ---
 "Test user agent processor with parameters":

modules/ingest-user-agent/src/yamlRestTest/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml

@@ -30,6 +30,5 @@
         id: 1
   - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" }
   - match: { _source.user_agent.name: "Test" }
-  - match: { _source.user_agent.device: {"name": "Other", "type": "Other" }}
   - is_false: _source.user_agent.os
   - is_false: _source.user_agent.version