Merge pull request #17263 from talevy/auto-convert

add  type conversion support to ConvertProcessor

core/src/main/java/org/elasticsearch/ingest/processor/ConvertProcessor.java

@@ -73,6 +73,23 @@ public final class ConvertProcessor extends AbstractProcessor {
             public Object convert(Object value) {
                 return value.toString();
             }
+        }, AUTO {
+            @Override
+            public Object convert(Object value) {
+                if (!(value instanceof String)) {
+                   return value;
+                }
+                try {
+                    return BOOLEAN.convert(value);
+                } catch (IllegalArgumentException e) { }
+                try {
+                    return INTEGER.convert(value);
+                } catch (IllegalArgumentException e) {}
+                try {
+                    return FLOAT.convert(value);
+                } catch (IllegalArgumentException e) {}
+                return value;
+            }
         };
 
         @Override
@@ -94,11 +111,13 @@ public final class ConvertProcessor extends AbstractProcessor {
     public static final String TYPE = "convert";
 
     private final String field;
+    private final String targetField;
     private final Type convertType;
 
-    ConvertProcessor(String tag, String field, Type convertType) {
+    ConvertProcessor(String tag, String field, String targetField, Type convertType) {
         super(tag);
         this.field = field;
+        this.targetField = targetField;
         this.convertType = convertType;
     }
 
@@ -106,6 +125,10 @@ public final class ConvertProcessor extends AbstractProcessor {
         return field;
     }
 
+    String getTargetField() {
+        return targetField;
+    }
+
     Type getConvertType() {
         return convertType;
     }
@@ -128,7 +151,7 @@ public final class ConvertProcessor extends AbstractProcessor {
         } else {
             newValue = convertType.convert(oldValue);
         }
-        document.setFieldValue(field, newValue);
+        document.setFieldValue(targetField, newValue);
     }
 
     @Override
@@ -141,8 +164,9 @@ public final class ConvertProcessor extends AbstractProcessor {
         public ConvertProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             String typeProperty = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "type");
+            String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", field);
             Type convertType = Type.fromString(processorTag, "type", typeProperty);
-            return new ConvertProcessor(processorTag, field, convertType);
+            return new ConvertProcessor(processorTag, field, targetField, convertType);
         }
     }
 }

core/src/test/java/org/elasticsearch/ingest/processor/ConvertProcessorFactoryTests.java

@@ -21,7 +21,6 @@ package org.elasticsearch.ingest.processor;
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ingest.core.AbstractProcessorFactory;
-import org.elasticsearch.ingest.core.Processor;
 import org.elasticsearch.test.ESTestCase;
 import org.hamcrest.Matchers;
 
@@ -44,6 +43,7 @@ public class ConvertProcessorFactoryTests extends ESTestCase {
         ConvertProcessor convertProcessor = factory.create(config);
         assertThat(convertProcessor.getTag(), equalTo(processorTag));
         assertThat(convertProcessor.getField(), equalTo("field1"));
+        assertThat(convertProcessor.getTargetField(), equalTo("field1"));
         assertThat(convertProcessor.getConvertType(), equalTo(type));
     }
 
@@ -88,4 +88,20 @@ public class ConvertProcessorFactoryTests extends ESTestCase {
             assertThat(e.getMessage(), Matchers.equalTo("[type] required property is missing"));
         }
     }
+
+    public void testCreateWithExplicitTargetField() throws Exception {
+        ConvertProcessor.Factory factory = new ConvertProcessor.Factory();
+        Map<String, Object> config = new HashMap<>();
+        ConvertProcessor.Type type = randomFrom(ConvertProcessor.Type.values());
+        config.put("field", "field1");
+        config.put("target_field", "field2");
+        config.put("type", type.toString());
+        String processorTag = randomAsciiOfLength(10);
+        config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
+        ConvertProcessor convertProcessor = factory.create(config);
+        assertThat(convertProcessor.getTag(), equalTo(processorTag));
+        assertThat(convertProcessor.getField(), equalTo("field1"));
+        assertThat(convertProcessor.getTargetField(), equalTo("field2"));
+        assertThat(convertProcessor.getConvertType(), equalTo(type));
+    }
 }

core/src/test/java/org/elasticsearch/ingest/processor/ConvertProcessorTests.java

@@ -34,6 +34,7 @@ import java.util.Map;
 import static org.elasticsearch.ingest.processor.ConvertProcessor.Type;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.sameInstance;
 
 public class ConvertProcessorTests extends ESTestCase {
 
@@ -41,7 +42,7 @@ public class ConvertProcessorTests extends ESTestCase {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
         int randomInt = randomInt();
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, randomInt);
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.INTEGER);
+        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.INTEGER);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(fieldName, Integer.class), equalTo(randomInt));
     }
@@ -57,7 +58,7 @@ public class ConvertProcessorTests extends ESTestCase {
             expectedList.add(randomInt);
         }
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.INTEGER);
+        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.INTEGER);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList));
     }
@@ -68,7 +69,7 @@ public class ConvertProcessorTests extends ESTestCase {
         String value = "string-" + randomAsciiOfLengthBetween(1, 10);
         ingestDocument.setFieldValue(fieldName, value);
 
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.INTEGER);
+        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.INTEGER);
         try {
             processor.execute(ingestDocument);
             fail("processor execute should have failed");
@@ -84,7 +85,7 @@ public class ConvertProcessorTests extends ESTestCase {
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, randomFloat);
         expectedResult.put(fieldName, randomFloat);
 
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.FLOAT);
+        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.FLOAT);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(fieldName, Float.class), equalTo(randomFloat));
     }
@@ -100,7 +101,7 @@ public class ConvertProcessorTests extends ESTestCase {
             expectedList.add(randomFloat);
         }
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.FLOAT);
+        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.FLOAT);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList));
     }
@@ -111,7 +112,7 @@ public class ConvertProcessorTests extends ESTestCase {
         String value = "string-" + randomAsciiOfLengthBetween(1, 10);
         ingestDocument.setFieldValue(fieldName, value);
 
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.FLOAT);
+        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.FLOAT);
         try {
             processor.execute(ingestDocument);
             fail("processor execute should have failed");
@@ -129,7 +130,7 @@ public class ConvertProcessorTests extends ESTestCase {
         }
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, booleanString);
 
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.BOOLEAN);
+        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.BOOLEAN);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(fieldName, Boolean.class), equalTo(randomBoolean));
     }
@@ -149,7 +150,7 @@ public class ConvertProcessorTests extends ESTestCase {
             expectedList.add(randomBoolean);
         }
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.BOOLEAN);
+        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.BOOLEAN);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList));
     }
@@ -166,7 +167,7 @@ public class ConvertProcessorTests extends ESTestCase {
         }
         ingestDocument.setFieldValue(fieldName, fieldValue);
 
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.BOOLEAN);
+        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.BOOLEAN);
         try {
             processor.execute(ingestDocument);
             fail("processor execute should have failed");
@@ -200,7 +201,7 @@ public class ConvertProcessorTests extends ESTestCase {
         }
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
 
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.STRING);
+        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.STRING);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(fieldName, String.class), equalTo(expectedFieldValue));
     }
@@ -236,7 +237,7 @@ public class ConvertProcessorTests extends ESTestCase {
             expectedList.add(randomValueString);
         }
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, Type.STRING);
+        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.STRING);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList));
     }
@@ -245,7 +246,7 @@ public class ConvertProcessorTests extends ESTestCase {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
         String fieldName = RandomDocumentPicks.randomFieldName(random());
         Type type = randomFrom(Type.values());
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, type);
+        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, type);
         try {
             processor.execute(ingestDocument);
             fail("processor execute should have failed");
@@ -257,7 +258,7 @@ public class ConvertProcessorTests extends ESTestCase {
     public void testConvertNullField() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null));
         Type type = randomFrom(Type.values());
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", type);
+        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", type);
         try {
             processor.execute(ingestDocument);
             fail("processor execute should have failed");
@@ -265,4 +266,80 @@ public class ConvertProcessorTests extends ESTestCase {
             assertThat(e.getMessage(), equalTo("Field [field] is null, cannot be converted to type [" + type + "]"));
         }
     }
+
+    public void testAutoConvertNotString() throws Exception {
+        Object randomValue;
+        switch(randomIntBetween(0, 2)) {
+            case 0:
+                float randomFloat = randomFloat();
+                randomValue = randomFloat;
+                break;
+            case 1:
+                int randomInt = randomInt();
+                randomValue = randomInt;
+                break;
+            case 2:
+                boolean randomBoolean = randomBoolean();
+                randomValue = randomBoolean;
+                break;
+            default:
+                throw new UnsupportedOperationException();
+        }
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", randomValue));
+        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO);
+        processor.execute(ingestDocument);
+        Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
+        assertThat(convertedValue, sameInstance(randomValue));
+    }
+
+    public void testAutoConvertStringNotMatched() throws Exception {
+        String value = "notAnIntFloatOrBool";
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", value));
+        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO);
+        processor.execute(ingestDocument);
+        Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
+        assertThat(convertedValue, sameInstance(value));
+    }
+
+    public void testAutoConvertMatchBoolean() throws Exception {
+        boolean randomBoolean = randomBoolean();
+        String booleanString = Boolean.toString(randomBoolean);
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", booleanString));
+        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO);
+        processor.execute(ingestDocument);
+        Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
+        assertThat(convertedValue, equalTo(randomBoolean));
+    }
+
+    public void testAutoConvertMatchInteger() throws Exception {
+        int randomInt = randomInt();
+        String randomString = Integer.toString(randomInt);
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", randomString));
+        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO);
+        processor.execute(ingestDocument);
+        Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
+        assertThat(convertedValue, equalTo(randomInt));
+    }
+
+    public void testAutoConvertMatchFloat() throws Exception {
+        float randomFloat = randomFloat();
+        String randomString = Float.toString(randomFloat);
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", randomString));
+        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO);
+        processor.execute(ingestDocument);
+        Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
+        assertThat(convertedValue, equalTo(randomFloat));
+    }
+
+    public void testTargetField() throws Exception {
+        IngestDocument ingestDocument = new IngestDocument(new HashMap<>(), new HashMap<>());
+        int randomInt = randomInt();
+        String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, String.valueOf(randomInt));
+        String targetField = fieldName + randomAsciiOfLength(5);
+        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, targetField, Type.INTEGER);
+        processor.execute(ingestDocument);
+        assertThat(ingestDocument.getFieldValue(fieldName, String.class), equalTo(String.valueOf(randomInt)));
+        assertThat(ingestDocument.getFieldValue(targetField, Integer.class), equalTo(randomInt));
+
+    }
 }

docs/reference/ingest/ingest-node.asciidoc

@@ -668,18 +668,25 @@ Accepts a single value or an array of values.
 Converts an existing field's value to a different type, such as converting a string to an integer.
 If the field value is an array, all members will be converted.
 
-The supported types include: `integer`, `float`, `string`, and `boolean`.
+The supported types include: `integer`, `float`, `string`, `boolean`, and `auto`.
 
 Specifying `boolean` will set the field to true if its string value is equal to `true` (ignore case), to
 false if its string value is equal to `false` (ignore case), or it will throw an exception otherwise.
 
+Specifying `auto` will attempt to convert the string-valued `field` into the closest non-string type.
+For example, a field whose value is `"true"` will be converted to its respective boolean type: `true`. And 
+a value of `"242.15"` will "automatically" be converted to `242.15` of type `float`. If a provided field cannot 
+be appropriately converted, the Convert Processor will still process successfully and leave the field value as-is. In 
+such a case, `target_field` will still be updated with the unconverted field value.
+
 [[convert-options]]
 .Convert Options
 [options="header"]
 |======
-| Name      | Required  | Default  | Description
-| `field`   | yes       | -        | The field whose value is to be converted
-| `type`    | yes       | -        | The type to convert the existing value to
+| Name           | Required  | Default  | Description
+| `field`        | yes       | -        | The field whose value is to be converted
+| `target_field` | no        | `field`  | The field to assign the converted value to, by default `field` is updated in-place
+| `type`         | yes       | -        | The type to convert the existing value to
 |======
 
 [source,js]