add ignore_missing option to relevant processors (#20194)

core/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentSimpleResultTests.java

@@ -27,7 +27,7 @@ import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 
-import static org.elasticsearch.ingest.IngestDocumentTests.assertIngestDocument;
+import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.instanceOf;
 

core/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java

@@ -34,7 +34,7 @@ import org.junit.Before;
 import java.util.Collections;
 import java.util.Map;
 
-import static org.elasticsearch.ingest.IngestDocumentTests.assertIngestDocument;
+import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.not;

core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestTests.java

@@ -20,22 +20,13 @@
 package org.elasticsearch.action.ingest;
 
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.ingest.IngestDocument;
-import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
 
-import static org.elasticsearch.ingest.IngestDocumentTests.assertIngestDocument;
 import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.instanceOf;
-import static org.hamcrest.CoreMatchers.nullValue;
 
 public class SimulatePipelineRequestTests extends ESTestCase {
 

core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineResponseTests.java

@@ -30,7 +30,7 @@ import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 
-import static org.elasticsearch.ingest.IngestDocumentTests.assertIngestDocument;
+import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.CoreMatchers.nullValue;

core/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java

@@ -27,7 +27,7 @@ import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 
-import static org.elasticsearch.ingest.IngestDocumentTests.assertIngestDocument;
+import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;

core/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java

@@ -34,7 +34,7 @@ import java.util.HashMap;
 import java.util.Map;
 
 import static org.elasticsearch.common.xcontent.ToXContent.EMPTY_PARAMS;
-import static org.elasticsearch.ingest.IngestDocumentTests.assertIngestDocument;
+import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;

core/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java

@@ -33,6 +33,7 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 
+import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
 import static org.hamcrest.Matchers.both;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.endsWith;
@@ -1004,34 +1005,4 @@ public class IngestDocumentTests extends ESTestCase {
         }
     }
 
-    public static void assertIngestDocument(Object a, Object b) {
-        if (a instanceof Map) {
-            Map<?, ?> mapA = (Map<?, ?>) a;
-            Map<?, ?> mapB = (Map<?, ?>) b;
-            for (Map.Entry<?, ?> entry : mapA.entrySet()) {
-                if (entry.getValue() instanceof List || entry.getValue() instanceof Map) {
-                    assertIngestDocument(entry.getValue(), mapB.get(entry.getKey()));
-                }
-            }
-        } else if (a instanceof List) {
-            List<?> listA = (List<?>) a;
-            List<?> listB = (List<?>) b;
-            for (int i = 0; i < listA.size(); i++) {
-                Object value = listA.get(i);
-                if (value instanceof List || value instanceof Map) {
-                    assertIngestDocument(value, listB.get(i));
-                }
-            }
-        } else if (a instanceof byte[]) {
-            assertArrayEquals((byte[]) a, (byte[])b);
-        } else if (a instanceof IngestDocument) {
-            IngestDocument docA = (IngestDocument) a;
-            IngestDocument docB = (IngestDocument) b;
-            assertIngestDocument(docA.getSourceAndMetadata(), docB.getSourceAndMetadata());
-            assertIngestDocument(docA.getIngestMetadata(), docB.getIngestMetadata());
-        } else {
-            String msg = String.format(Locale.ROOT, "Expected %s class to be equal to %s", a.getClass().getName(), b.getClass().getName());
-            assertThat(msg, a, equalTo(b));
-        }
-    }
 }

docs/reference/ingest/ingest-node.asciidoc

@@ -706,10 +706,11 @@ such a case, `target_field` will still be updated with the unconverted field val
 .Convert Options
 [options="header"]
 |======
-| Name           | Required  | Default  | Description
-| `field`        | yes       | -        | The field whose value is to be converted
-| `target_field` | no        | `field`  | The field to assign the converted value to, by default `field` is updated in-place
-| `type`         | yes       | -        | The type to convert the existing value to
+| Name             | Required  | Default  | Description
+| `field`          | yes       | -        | The field whose value is to be converted
+| `target_field`   | no        | `field`  | The field to assign the converted value to, by default `field` is updated in-place
+| `type`           | yes       | -        | The type to convert the existing value to
+| `ignore_missing` | no        | `false`  | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document
 |======
 
 [source,js]
@@ -1142,6 +1143,7 @@ Grok expression.
 | `patterns`             | yes       | -                   | An ordered list of grok expression to match and extract named captures with. Returns on the first expression in the list that matches.
 | `pattern_definitions`  | no        | -                   | A map of pattern-name and pattern tuples defining custom patterns to be used by the current processor. Patterns matching existing names will override the pre-existing definition.
 | `trace_match`          | no        | false               | when true, `_ingest._grok_match_index` will be inserted into your matched document's metadata with the index into the pattern found in `patterns` that matched.
+| `ignore_missing`       | no        | false               | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document
 |======
 
 Here is an example of using the provided patterns to extract out and name structured fields from a string field in
@@ -1278,8 +1280,9 @@ Converts a string to its lowercase equivalent.
 .Lowercase Options
 [options="header"]
 |======
-| Name     | Required  | Default  | Description
-| `field`  | yes       | -        | The field to make lowercase
+| Name             | Required  | Default  | Description
+| `field`          | yes       | -        | The field to make lowercase
+| `ignore_missing` | no        | `false`  | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document
 |======
 
 [source,js]
@@ -1320,9 +1323,10 @@ Renames an existing field. If the field doesn't exist or the new name is already
 .Rename Options
 [options="header"]
 |======
-| Name           | Required  | Default  | Description
-| `field`        | yes       | -        | The field to be renamed
-| `target_field` | yes       | -        | The new name of the field
+| Name             | Required  | Default  | Description
+| `field`          | yes       | -        | The field to be renamed
+| `target_field`   | yes       | -        | The new name of the field
+| `ignore_missing` | no        | `false`  | If `true` and `field` does not exist, the processor quietly exits without modifying the document
 |======
 
 [source,js]
@@ -1462,8 +1466,9 @@ NOTE: This only works on leading and trailing whitespace.
 .Trim Options
 [options="header"]
 |======
-| Name     | Required  | Default  | Description
-| `field`  | yes       | -        | The string-valued field to trim whitespace from
+| Name              | Required  | Default  | Description
+| `field`           | yes       | -        | The string-valued field to trim whitespace from
+| `ignore_missing`  | no        | `false`  | If `true` and `field` does not exist, the processor quietly exits without modifying the document
 |======
 
 [source,js]
@@ -1483,8 +1488,9 @@ Converts a string to its uppercase equivalent.
 .Uppercase Options
 [options="header"]
 |======
-| Name     | Required  | Default  | Description
-| `field`  | yes       | -        | The field to make uppercase
+| Name             | Required  | Default  | Description
+| `field`          | yes       | -        | The field to make uppercase
+| `ignore_missing` | no        | `false`  | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document
 |======
 
 [source,js]

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java

@@ -32,22 +32,40 @@ import java.util.Map;
  */
 abstract class AbstractStringProcessor extends AbstractProcessor {
     private final String field;
+    private final boolean ignoreMissing;
 
-    protected AbstractStringProcessor(String tag, String field) {
+    protected AbstractStringProcessor(String tag, String field, boolean ignoreMissing) {
         super(tag);
         this.field = field;
+        this.ignoreMissing = ignoreMissing;
     }
 
     public String getField() {
         return field;
     }
 
+    boolean isIgnoreMissing() {
+        return ignoreMissing;
+    }
+
     @Override
     public final void execute(IngestDocument document) {
-        String val = document.getFieldValue(field, String.class);
-        if (val == null) {
+        String val;
+
+        try {
+            val = document.getFieldValue(field, String.class);
+        } catch (IllegalArgumentException e) {
+            if (ignoreMissing && document.hasField(field) != true) {
+                return;
+            }
+            throw e;
+        }
+        if (val == null && ignoreMissing) {
+            return;
+        } else if (val == null) {
             throw new IllegalArgumentException("field [" + field + "] is null, cannot process it.");
         }
+
         document.setFieldValue(field, process(val));
     }
 
@@ -64,9 +82,10 @@ abstract class AbstractStringProcessor extends AbstractProcessor {
         public AbstractStringProcessor create(Map<String, Processor.Factory> registry, String tag,
                                               Map<String, Object> config) throws Exception {
             String field = ConfigurationUtils.readStringProperty(processorType, tag, config, "field");
-            return newProcessor(tag, field);
+            boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(processorType, tag, config, "ignore_missing", false);
+            return newProcessor(tag, field, ignoreMissing);
         }
 
-        protected abstract AbstractStringProcessor newProcessor(String processorTag, String field);
+        protected abstract AbstractStringProcessor newProcessor(String processorTag, String field, boolean ignoreMissing);
     }
 }

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java

@@ -114,12 +114,14 @@ public final class ConvertProcessor extends AbstractProcessor {
     private final String field;
     private final String targetField;
     private final Type convertType;
+    private final boolean ignoreMissing;
 
-    ConvertProcessor(String tag, String field, String targetField, Type convertType) {
+    ConvertProcessor(String tag, String field, String targetField, Type convertType, boolean ignoreMissing) {
         super(tag);
         this.field = field;
         this.targetField = targetField;
         this.convertType = convertType;
+        this.ignoreMissing = ignoreMissing;
     }
 
     String getField() {
@@ -134,11 +136,27 @@ public final class ConvertProcessor extends AbstractProcessor {
         return convertType;
     }
 
+    boolean isIgnoreMissing() {
+        return ignoreMissing;
+    }
+
     @Override
     public void execute(IngestDocument document) {
-        Object oldValue = document.getFieldValue(field, Object.class);
+        Object oldValue = null;
         Object newValue;
-        if (oldValue == null) {
+
+        try {
+            oldValue = document.getFieldValue(field, Object.class);
+        } catch (IllegalArgumentException e) {
+            if (ignoreMissing) {
+                return;
+            }
+            throw e;
+        }
+
+        if (oldValue == null && ignoreMissing) {
+            return;
+        } else if (oldValue == null) {
             throw new IllegalArgumentException("Field [" + field + "] is null, cannot be converted to type [" + convertType + "]");
         }
 
@@ -168,7 +186,8 @@ public final class ConvertProcessor extends AbstractProcessor {
             String typeProperty = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "type");
             String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", field);
             Type convertType = Type.fromString(processorTag, "type", typeProperty);
-            return new ConvertProcessor(processorTag, field, targetField, convertType);
+            boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
+            return new ConvertProcessor(processorTag, field, targetField, convertType, ignoreMissing);
         }
     }
 }

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java

@@ -39,21 +39,36 @@ public final class GrokProcessor extends AbstractProcessor {
     private final String matchField;
     private final Grok grok;
     private final boolean traceMatch;
+    private final boolean ignoreMissing;
 
-    public GrokProcessor(String tag, Map<String, String> patternBank, List<String> matchPatterns, String matchField) {
-        this(tag, patternBank, matchPatterns, matchField, false);
-    }
-
-    public GrokProcessor(String tag, Map<String, String> patternBank, List<String> matchPatterns, String matchField, boolean traceMatch) {
+    public GrokProcessor(String tag, Map<String, String> patternBank, List<String> matchPatterns, String matchField,
+                         boolean traceMatch, boolean ignoreMissing) {
         super(tag);
         this.matchField = matchField;
         this.grok = new Grok(patternBank, combinePatterns(matchPatterns, traceMatch));
         this.traceMatch = traceMatch;
+        this.ignoreMissing = ignoreMissing;
     }
 
     @Override
     public void execute(IngestDocument ingestDocument) throws Exception {
-        String fieldValue = ingestDocument.getFieldValue(matchField, String.class);
+        String fieldValue;
+
+        try {
+            fieldValue = ingestDocument.getFieldValue(matchField, String.class);
+        } catch (IllegalArgumentException e) {
+            if (ignoreMissing && ingestDocument.hasField(matchField) != true) {
+                return;
+            }
+            throw e;
+        }
+
+        if (fieldValue == null && ignoreMissing) {
+            return;
+        } else if (fieldValue == null) {
+            throw new IllegalArgumentException("field [" + matchField + "] is null, cannot process it.");
+        }
+
         Map<String, Object> matches = grok.captures(fieldValue);
         if (matches == null) {
             throw new IllegalArgumentException("Provided Grok expressions do not match field value: [" + fieldValue + "]");
@@ -77,10 +92,14 @@ public final class GrokProcessor extends AbstractProcessor {
         return TYPE;
     }
 
-    public Grok getGrok() {
+    Grok getGrok() {
         return grok;
     }
 
+    boolean isIgnoreMissing() {
+        return ignoreMissing;
+    }
+
     String getMatchField() {
         return matchField;
     }
@@ -128,6 +147,7 @@ public final class GrokProcessor extends AbstractProcessor {
             String matchField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             List<String> matchPatterns = ConfigurationUtils.readList(TYPE, processorTag, config, "patterns");
             boolean traceMatch = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "trace_match", false);
+            boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
 
             if (matchPatterns.isEmpty()) {
                 throw newConfigurationException(TYPE, processorTag, "patterns", "List of patterns must not be empty");
@@ -139,7 +159,7 @@ public final class GrokProcessor extends AbstractProcessor {
             }
 
             try {
-                return new GrokProcessor(processorTag, patternBank, matchPatterns, matchField, traceMatch);
+                return new GrokProcessor(processorTag, patternBank, matchPatterns, matchField, traceMatch, ignoreMissing);
             } catch (Exception e) {
                 throw newConfigurationException(TYPE, processorTag, "patterns",
                     "Invalid regex pattern found in: " + matchPatterns + ". " + e.getMessage());

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java

@@ -30,8 +30,8 @@ public final class LowercaseProcessor extends AbstractStringProcessor {
 
     public static final String TYPE = "lowercase";
 
-    LowercaseProcessor(String processorTag, String field) {
-        super(processorTag, field);
+    LowercaseProcessor(String processorTag, String field, boolean ignoreMissing) {
+        super(processorTag, field, ignoreMissing);
     }
 
     @Override
@@ -51,8 +51,8 @@ public final class LowercaseProcessor extends AbstractStringProcessor {
         }
 
         @Override
-        protected LowercaseProcessor newProcessor(String tag, String field) {
-            return new LowercaseProcessor(tag, field);
+        protected LowercaseProcessor newProcessor(String tag, String field, boolean ignoreMissing) {
+            return new LowercaseProcessor(tag, field, ignoreMissing);
         }
     }
 }

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java

@@ -35,11 +35,13 @@ public final class RenameProcessor extends AbstractProcessor {
 
     private final String field;
     private final String targetField;
+    private final boolean ignoreMissing;
 
-    RenameProcessor(String tag, String field, String targetField) {
+    RenameProcessor(String tag, String field, String targetField, boolean ignoreMissing) {
         super(tag);
         this.field = field;
         this.targetField = targetField;
+        this.ignoreMissing = ignoreMissing;
     }
 
     String getField() {
@@ -50,10 +52,18 @@ public final class RenameProcessor extends AbstractProcessor {
         return targetField;
     }
 
+    boolean isIgnoreMissing() {
+        return ignoreMissing;
+    }
+
     @Override
     public void execute(IngestDocument document) {
         if (document.hasField(field, true) == false) {
-            throw new IllegalArgumentException("field [" + field + "] doesn't exist");
+            if (ignoreMissing) {
+                return;
+            } else {
+                throw new IllegalArgumentException("field [" + field + "] doesn't exist");
+            }
         }
         // We fail here if the target field point to an array slot that is out of range.
         // If we didn't do this then we would fail if we set the value in the target_field
@@ -85,7 +95,8 @@ public final class RenameProcessor extends AbstractProcessor {
                                       Map<String, Object> config) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field");
-            return new RenameProcessor(processorTag, field, targetField);
+            boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
+            return new RenameProcessor(processorTag, field, targetField, ignoreMissing);
         }
     }
 }

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TrimProcessor.java

@@ -27,8 +27,8 @@ public final class TrimProcessor extends AbstractStringProcessor {
 
     public static final String TYPE = "trim";
 
-    TrimProcessor(String processorTag, String field) {
-        super(processorTag, field);
+    TrimProcessor(String processorTag, String field, boolean ignoreMissing) {
+        super(processorTag, field, ignoreMissing);
     }
 
     @Override
@@ -48,8 +48,8 @@ public final class TrimProcessor extends AbstractStringProcessor {
         }
 
         @Override
-        protected TrimProcessor newProcessor(String tag, String field) {
-            return new TrimProcessor(tag, field);
+        protected TrimProcessor newProcessor(String tag, String field, boolean ignoreMissing) {
+            return new TrimProcessor(tag, field, ignoreMissing);
         }
     }
 }

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java

@@ -29,8 +29,8 @@ public final class UppercaseProcessor extends AbstractStringProcessor {
 
     public static final String TYPE = "uppercase";
 
-    UppercaseProcessor(String processorTag, String field) {
-        super(processorTag, field);
+    UppercaseProcessor(String processorTag, String field, boolean ignoreMissing) {
+        super(processorTag, field, ignoreMissing);
     }
 
     @Override
@@ -50,8 +50,8 @@ public final class UppercaseProcessor extends AbstractStringProcessor {
         }
 
         @Override
-        protected UppercaseProcessor newProcessor(String tag, String field) {
-            return new UppercaseProcessor(tag, field);
+        protected UppercaseProcessor newProcessor(String tag, String field, boolean ignoreMissing) {
+            return new UppercaseProcessor(tag, field, ignoreMissing);
         }
     }
 }

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorTestCase.java

@@ -27,12 +27,13 @@ import org.elasticsearch.test.ESTestCase;
 import java.util.Collections;
 import java.util.HashMap;
 
+import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 
 public abstract class AbstractStringProcessorTestCase extends ESTestCase {
 
-    protected abstract AbstractStringProcessor newProcessor(String field);
+    protected abstract AbstractStringProcessor newProcessor(String field, boolean ignoreMissing);
 
     protected String modifyInput(String input) {
         return input;
@@ -44,45 +45,60 @@ public abstract class AbstractStringProcessorTestCase extends ESTestCase {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
         String fieldValue = RandomDocumentPicks.randomString(random());
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, modifyInput(fieldValue));
-        Processor processor = newProcessor(fieldName);
+        Processor processor = newProcessor(fieldName, randomBoolean());
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(fieldName, String.class), equalTo(expectedResult(fieldValue)));
     }
 
     public void testFieldNotFound() throws Exception {
         String fieldName = RandomDocumentPicks.randomFieldName(random());
-        Processor processor = newProcessor(fieldName);
+        Processor processor = newProcessor(fieldName, false);
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
-        try {
-            processor.execute(ingestDocument);
-            fail("processor should have failed");
-        } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), containsString("not present as part of path [" + fieldName + "]"));
-        }
+        Exception e = expectThrows(Exception.class, () -> processor.execute(ingestDocument));
+        assertThat(e.getMessage(), containsString("not present as part of path [" + fieldName + "]"));
+    }
+
+    public void testFieldNotFoundWithIgnoreMissing() throws Exception {
+        String fieldName = RandomDocumentPicks.randomFieldName(random());
+        Processor processor = newProcessor(fieldName, true);
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        processor.execute(ingestDocument);
+        assertIngestDocument(originalIngestDocument, ingestDocument);
     }
 
     public void testNullValue() throws Exception {
-        Processor processor = newProcessor("field");
+        Processor processor = newProcessor("field", false);
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null));
-        try {
-            processor.execute(ingestDocument);
-            fail("processor should have failed");
-        } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), equalTo("field [field] is null, cannot process it."));
-        }
+        Exception e = expectThrows(Exception.class, () -> processor.execute(ingestDocument));
+        assertThat(e.getMessage(), equalTo("field [field] is null, cannot process it."));
+    }
+
+    public void testNullValueWithIgnoreMissing() throws Exception {
+        Processor processor = newProcessor("field", true);
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null));
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        processor.execute(ingestDocument);
+        assertIngestDocument(originalIngestDocument, ingestDocument);
     }
 
     public void testNonStringValue() throws Exception {
         String fieldName = RandomDocumentPicks.randomFieldName(random());
-        Processor processor = newProcessor(fieldName);
+        Processor processor = newProcessor(fieldName, false);
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
+        ingestDocument.setFieldValue(fieldName, randomInt());
+        Exception e = expectThrows(Exception.class, () -> processor.execute(ingestDocument));
+        assertThat(e.getMessage(), equalTo("field [" + fieldName +
+            "] of type [java.lang.Integer] cannot be cast to [java.lang.String]"));
+    }
+
+    public void testNonStringValueWithIgnoreMissing() throws Exception {
+        String fieldName = RandomDocumentPicks.randomFieldName(random());
+        Processor processor = newProcessor(fieldName, true);
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
         ingestDocument.setFieldValue(fieldName, randomInt());
-        try {
-            processor.execute(ingestDocument);
-            fail("processor should have failed");
-        } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), equalTo("field [" + fieldName +
-                    "] of type [java.lang.Integer] cannot be cast to [java.lang.String]"));
-        }
+        Exception e = expectThrows(Exception.class, () -> processor.execute(ingestDocument));
+        assertThat(e.getMessage(), equalTo("field [" + fieldName +
+            "] of type [java.lang.Integer] cannot be cast to [java.lang.String]"));
     }
 }

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorFactoryTests.java

@@ -27,6 +27,7 @@ import java.util.HashMap;
 import java.util.Map;
 
 import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
 
 public class ConvertProcessorFactoryTests extends ESTestCase {
@@ -43,6 +44,7 @@ public class ConvertProcessorFactoryTests extends ESTestCase {
         assertThat(convertProcessor.getField(), equalTo("field1"));
         assertThat(convertProcessor.getTargetField(), equalTo("field1"));
         assertThat(convertProcessor.getConvertType(), equalTo(type));
+        assertThat(convertProcessor.isIgnoreMissing(), is(false));
     }
 
     public void testCreateUnsupportedType() throws Exception {
@@ -100,5 +102,22 @@ public class ConvertProcessorFactoryTests extends ESTestCase {
         assertThat(convertProcessor.getField(), equalTo("field1"));
         assertThat(convertProcessor.getTargetField(), equalTo("field2"));
         assertThat(convertProcessor.getConvertType(), equalTo(type));
+        assertThat(convertProcessor.isIgnoreMissing(), is(false));
+    }
+
+    public void testCreateWithIgnoreMissing() throws Exception {
+        ConvertProcessor.Factory factory = new ConvertProcessor.Factory();
+        Map<String, Object> config = new HashMap<>();
+        ConvertProcessor.Type type = randomFrom(ConvertProcessor.Type.values());
+        config.put("field", "field1");
+        config.put("type", type.toString());
+        config.put("ignore_missing", true);
+        String processorTag = randomAsciiOfLength(10);
+        ConvertProcessor convertProcessor = factory.create(null, processorTag, config);
+        assertThat(convertProcessor.getTag(), equalTo(processorTag));
+        assertThat(convertProcessor.getField(), equalTo("field1"));
+        assertThat(convertProcessor.getTargetField(), equalTo("field1"));
+        assertThat(convertProcessor.getConvertType(), equalTo(type));
+        assertThat(convertProcessor.isIgnoreMissing(), is(true));
     }
 }

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java

@@ -31,6 +31,7 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 
+import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
 import static org.elasticsearch.ingest.common.ConvertProcessor.Type;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -42,7 +43,7 @@ public class ConvertProcessorTests extends ESTestCase {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
         int randomInt = randomInt();
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, randomInt);
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.INTEGER);
+        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.INTEGER, false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(fieldName, Integer.class), equalTo(randomInt));
     }
@@ -58,7 +59,7 @@ public class ConvertProcessorTests extends ESTestCase {
             expectedList.add(randomInt);
         }
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.INTEGER);
+        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.INTEGER, false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList));
     }
@@ -69,7 +70,7 @@ public class ConvertProcessorTests extends ESTestCase {
         String value = "string-" + randomAsciiOfLengthBetween(1, 10);
         ingestDocument.setFieldValue(fieldName, value);
 
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.INTEGER);
+        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.INTEGER, false);
         try {
             processor.execute(ingestDocument);
             fail("processor execute should have failed");
@@ -85,7 +86,7 @@ public class ConvertProcessorTests extends ESTestCase {
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, randomFloat);
         expectedResult.put(fieldName, randomFloat);
 
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.FLOAT);
+        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.FLOAT, false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(fieldName, Float.class), equalTo(randomFloat));
     }
@@ -101,7 +102,7 @@ public class ConvertProcessorTests extends ESTestCase {
             expectedList.add(randomFloat);
         }
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.FLOAT);
+        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.FLOAT, false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList));
     }
@@ -112,7 +113,7 @@ public class ConvertProcessorTests extends ESTestCase {
         String value = "string-" + randomAsciiOfLengthBetween(1, 10);
         ingestDocument.setFieldValue(fieldName, value);
 
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.FLOAT);
+        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.FLOAT, false);
         try {
             processor.execute(ingestDocument);
             fail("processor execute should have failed");
@@ -130,7 +131,7 @@ public class ConvertProcessorTests extends ESTestCase {
         }
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, booleanString);
 
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.BOOLEAN);
+        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.BOOLEAN, false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(fieldName, Boolean.class), equalTo(randomBoolean));
     }
@@ -150,7 +151,7 @@ public class ConvertProcessorTests extends ESTestCase {
             expectedList.add(randomBoolean);
         }
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.BOOLEAN);
+        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.BOOLEAN, false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList));
     }
@@ -167,7 +168,7 @@ public class ConvertProcessorTests extends ESTestCase {
         }
         ingestDocument.setFieldValue(fieldName, fieldValue);
 
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.BOOLEAN);
+        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.BOOLEAN, false);
         try {
             processor.execute(ingestDocument);
             fail("processor execute should have failed");
@@ -201,7 +202,7 @@ public class ConvertProcessorTests extends ESTestCase {
         }
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
 
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.STRING);
+        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.STRING, false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(fieldName, String.class), equalTo(expectedFieldValue));
     }
@@ -237,7 +238,7 @@ public class ConvertProcessorTests extends ESTestCase {
             expectedList.add(randomValueString);
         }
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.STRING);
+        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.STRING, false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList));
     }
@@ -246,7 +247,7 @@ public class ConvertProcessorTests extends ESTestCase {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
         String fieldName = RandomDocumentPicks.randomFieldName(random());
         Type type = randomFrom(Type.values());
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, type);
+        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, type, false);
         try {
             processor.execute(ingestDocument);
             fail("processor execute should have failed");
@@ -258,7 +259,7 @@ public class ConvertProcessorTests extends ESTestCase {
     public void testConvertNullField() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null));
         Type type = randomFrom(Type.values());
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", type);
+        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", type, false);
         try {
             processor.execute(ingestDocument);
             fail("processor execute should have failed");
@@ -267,6 +268,25 @@ public class ConvertProcessorTests extends ESTestCase {
         }
     }
 
+    public void testConvertNonExistingFieldWithIgnoreMissing() throws Exception {
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        String fieldName = RandomDocumentPicks.randomFieldName(random());
+        Type type = randomFrom(Type.values());
+        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, type, true);
+        processor.execute(ingestDocument);
+        assertIngestDocument(originalIngestDocument, ingestDocument);
+    }
+
+    public void testConvertNullFieldWithIgnoreMissing() throws Exception {
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null));
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        Type type = randomFrom(Type.values());
+        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", type, true);
+        processor.execute(ingestDocument);
+        assertIngestDocument(originalIngestDocument, ingestDocument);
+    }
+
     public void testAutoConvertNotString() throws Exception {
         Object randomValue;
         switch(randomIntBetween(0, 2)) {
@@ -286,7 +306,7 @@ public class ConvertProcessorTests extends ESTestCase {
                 throw new UnsupportedOperationException();
         }
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", randomValue));
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO);
+        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO, false);
         processor.execute(ingestDocument);
         Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
         assertThat(convertedValue, sameInstance(randomValue));
@@ -295,7 +315,7 @@ public class ConvertProcessorTests extends ESTestCase {
     public void testAutoConvertStringNotMatched() throws Exception {
         String value = "notAnIntFloatOrBool";
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", value));
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO);
+        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO, false);
         processor.execute(ingestDocument);
         Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
         assertThat(convertedValue, sameInstance(value));
@@ -306,7 +326,7 @@ public class ConvertProcessorTests extends ESTestCase {
         String booleanString = Boolean.toString(randomBoolean);
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(),
             Collections.singletonMap("field", booleanString));
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO);
+        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO, false);
         processor.execute(ingestDocument);
         Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
         assertThat(convertedValue, equalTo(randomBoolean));
@@ -316,7 +336,7 @@ public class ConvertProcessorTests extends ESTestCase {
         int randomInt = randomInt();
         String randomString = Integer.toString(randomInt);
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", randomString));
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO);
+        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO, false);
         processor.execute(ingestDocument);
         Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
         assertThat(convertedValue, equalTo(randomInt));
@@ -326,7 +346,7 @@ public class ConvertProcessorTests extends ESTestCase {
         float randomFloat = randomFloat();
         String randomString = Float.toString(randomFloat);
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", randomString));
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO);
+        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), "field", "field", Type.AUTO, false);
         processor.execute(ingestDocument);
         Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
         assertThat(convertedValue, equalTo(randomFloat));
@@ -337,10 +357,9 @@ public class ConvertProcessorTests extends ESTestCase {
         int randomInt = randomInt();
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, String.valueOf(randomInt));
         String targetField = fieldName + randomAsciiOfLength(5);
-        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, targetField, Type.INTEGER);
+        Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, targetField, Type.INTEGER, false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(fieldName, String.class), equalTo(String.valueOf(randomInt)));
         assertThat(ingestDocument.getFieldValue(targetField, Integer.class), equalTo(randomInt));
-
     }
 }

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java

@@ -86,7 +86,7 @@ public class DotExpanderProcessorTests extends ESTestCase {
         // so because foo is no branch field but a value field the `foo.bar` field can't be expanded
         // into [foo].[bar], so foo should be renamed first into `[foo].[bar]:
         IngestDocument document = new IngestDocument(source, Collections.emptyMap());
-        Processor processor = new RenameProcessor("_tag", "foo", "foo.bar");
+        Processor processor = new RenameProcessor("_tag", "foo", "foo.bar", false);
         processor.execute(document);
         processor = new DotExpanderProcessor("_tag", null, "foo.bar");
         processor.execute(document);

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorTests.java

@@ -49,7 +49,7 @@ public class ForEachProcessorTests extends ESTestCase {
         );
 
         ForEachProcessor processor = new ForEachProcessor(
-            "_tag", "values", new UppercaseProcessor("_tag", "_ingest._value")
+            "_tag", "values", new UppercaseProcessor("_tag", "_ingest._value", false)
         );
         processor.execute(ingestDocument);
 
@@ -197,7 +197,7 @@ public class ForEachProcessorTests extends ESTestCase {
 
         ForEachProcessor processor = new ForEachProcessor(
                 "_tag", "values", new CompoundProcessor(false,
-                Collections.singletonList(new UppercaseProcessor("_tag_upper", "_ingest._value")),
+                Collections.singletonList(new UppercaseProcessor("_tag_upper", "_ingest._value", false)),
                 Collections.singletonList(new AppendProcessor("_tag",
                         ts.compile("errors"), (model) -> (Collections.singletonList("added"))))
         ));

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorFactoryTests.java

@@ -27,6 +27,7 @@ import java.util.HashMap;
 import java.util.Map;
 
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
 
 public class GrokProcessorFactoryTests extends ESTestCase {
@@ -42,6 +43,22 @@ public class GrokProcessorFactoryTests extends ESTestCase {
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getMatchField(), equalTo("_field"));
         assertThat(processor.getGrok(), notNullValue());
+        assertThat(processor.isIgnoreMissing(), is(false));
+    }
+
+    public void testBuildWithIgnoreMissing() throws Exception {
+        GrokProcessor.Factory factory = new GrokProcessor.Factory(Collections.emptyMap());
+
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "_field");
+        config.put("patterns", Collections.singletonList("(?<foo>\\w+)"));
+        config.put("ignore_missing", true);
+        String processorTag = randomAsciiOfLength(10);
+        GrokProcessor processor = factory.create(null, processorTag, config);
+        assertThat(processor.getTag(), equalTo(processorTag));
+        assertThat(processor.getMatchField(), equalTo("_field"));
+        assertThat(processor.getGrok(), notNullValue());
+        assertThat(processor.isIgnoreMissing(), is(true));
     }
 
     public void testBuildMissingField() throws Exception {

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorTests.java

@@ -28,6 +28,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
+import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
 import static org.hamcrest.Matchers.equalTo;
 
 
@@ -38,7 +39,7 @@ public class GrokProcessorTests extends ESTestCase {
         IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
         doc.setFieldValue(fieldName, "1");
         GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), Collections.singletonMap("ONE", "1"),
-            Collections.singletonList("%{ONE:one}"), fieldName);
+            Collections.singletonList("%{ONE:one}"), fieldName, false, false);
         processor.execute(doc);
         assertThat(doc.getFieldValue("one", String.class), equalTo("1"));
     }
@@ -48,7 +49,7 @@ public class GrokProcessorTests extends ESTestCase {
         IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
         doc.setFieldValue(fieldName, "23");
         GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), Collections.singletonMap("ONE", "1"),
-            Collections.singletonList("%{ONE:one}"), fieldName);
+            Collections.singletonList("%{ONE:one}"), fieldName, false, false);
         Exception e = expectThrows(Exception.class, () -> processor.execute(doc));
         assertThat(e.getMessage(), equalTo("Provided Grok expressions do not match field value: [23]"));
     }
@@ -59,17 +60,48 @@ public class GrokProcessorTests extends ESTestCase {
         originalDoc.setFieldValue(fieldName, fieldName);
         IngestDocument doc = new IngestDocument(originalDoc);
         GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), Collections.emptyMap(),
-            Collections.singletonList(fieldName), fieldName);
+            Collections.singletonList(fieldName), fieldName, false, false);
         processor.execute(doc);
         assertThat(doc, equalTo(originalDoc));
     }
 
+    public void testNullField() {
+        String fieldName = RandomDocumentPicks.randomFieldName(random());
+        IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
+        doc.setFieldValue(fieldName, null);
+        GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), Collections.singletonMap("ONE", "1"),
+            Collections.singletonList("%{ONE:one}"), fieldName, false, false);
+        Exception e = expectThrows(Exception.class, () -> processor.execute(doc));
+        assertThat(e.getMessage(), equalTo("field [" + fieldName + "] is null, cannot process it."));
+    }
+
+    public void testNullFieldWithIgnoreMissing() throws Exception {
+        String fieldName = RandomDocumentPicks.randomFieldName(random());
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
+        originalIngestDocument.setFieldValue(fieldName, null);
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), Collections.singletonMap("ONE", "1"),
+            Collections.singletonList("%{ONE:one}"), fieldName, false, true);
+        processor.execute(ingestDocument);
+        assertIngestDocument(originalIngestDocument, ingestDocument);
+    }
+
     public void testNotStringField() {
         String fieldName = RandomDocumentPicks.randomFieldName(random());
         IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
         doc.setFieldValue(fieldName, 1);
         GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), Collections.singletonMap("ONE", "1"),
-            Collections.singletonList("%{ONE:one}"), fieldName);
+            Collections.singletonList("%{ONE:one}"), fieldName, false, false);
+        Exception e = expectThrows(Exception.class, () -> processor.execute(doc));
+        assertThat(e.getMessage(), equalTo("field [" + fieldName + "] of type [java.lang.Integer] cannot be cast to [java.lang.String]"));
+    }
+
+    public void testNotStringFieldWithIgnoreMissing() {
+        String fieldName = RandomDocumentPicks.randomFieldName(random());
+        IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
+        doc.setFieldValue(fieldName, 1);
+        GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), Collections.singletonMap("ONE", "1"),
+            Collections.singletonList("%{ONE:one}"), fieldName, false, true);
         Exception e = expectThrows(Exception.class, () -> processor.execute(doc));
         assertThat(e.getMessage(), equalTo("field [" + fieldName + "] of type [java.lang.Integer] cannot be cast to [java.lang.String]"));
     }
@@ -78,11 +110,21 @@ public class GrokProcessorTests extends ESTestCase {
         String fieldName = "foo.bar";
         IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
         GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), Collections.singletonMap("ONE", "1"),
-            Collections.singletonList("%{ONE:one}"), fieldName);
+            Collections.singletonList("%{ONE:one}"), fieldName, false, false);
         Exception e = expectThrows(Exception.class, () -> processor.execute(doc));
         assertThat(e.getMessage(), equalTo("field [foo] not present as part of path [foo.bar]"));
     }
 
+    public void testMissingFieldWithIgnoreMissing() throws Exception {
+        String fieldName = "foo.bar";
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
+        GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), Collections.singletonMap("ONE", "1"),
+            Collections.singletonList("%{ONE:one}"), fieldName, false, true);
+        processor.execute(ingestDocument);
+        assertIngestDocument(originalIngestDocument, ingestDocument);
+    }
+
     public void testMultiplePatternsWithMatchReturn() throws Exception {
         String fieldName = RandomDocumentPicks.randomFieldName(random());
         IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
@@ -92,7 +134,7 @@ public class GrokProcessorTests extends ESTestCase {
         patternBank.put("TWO", "2");
         patternBank.put("THREE", "3");
         GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), patternBank,
-            Arrays.asList("%{ONE:one}", "%{TWO:two}", "%{THREE:three}"), fieldName);
+            Arrays.asList("%{ONE:one}", "%{TWO:two}", "%{THREE:three}"), fieldName, false, false);
         processor.execute(doc);
         assertThat(doc.hasField("one"), equalTo(false));
         assertThat(doc.getFieldValue("two", String.class), equalTo("2"));
@@ -108,7 +150,7 @@ public class GrokProcessorTests extends ESTestCase {
         patternBank.put("TWO", "2");
         patternBank.put("THREE", "3");
         GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), patternBank,
-            Arrays.asList("%{ONE:one}", "%{TWO:two}", "%{THREE:three}"), fieldName, true);
+            Arrays.asList("%{ONE:one}", "%{TWO:two}", "%{THREE:three}"), fieldName, true, false);
         processor.execute(doc);
         assertThat(doc.hasField("one"), equalTo(false));
         assertThat(doc.getFieldValue("two", String.class), equalTo("2"));

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/LowercaseProcessorFactoryTests.java

@@ -26,6 +26,7 @@ import java.util.HashMap;
 import java.util.Map;
 
 import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.is;
 
 public class LowercaseProcessorFactoryTests extends ESTestCase {
 
@@ -37,6 +38,19 @@ public class LowercaseProcessorFactoryTests extends ESTestCase {
         LowercaseProcessor uppercaseProcessor = (LowercaseProcessor)factory.create(null, processorTag, config);
         assertThat(uppercaseProcessor.getTag(), equalTo(processorTag));
         assertThat(uppercaseProcessor.getField(), equalTo("field1"));
+        assertThat(uppercaseProcessor.isIgnoreMissing(), is(false));
+    }
+
+    public void testCreateWithIgnoreMissing() throws Exception {
+        LowercaseProcessor.Factory factory = new LowercaseProcessor.Factory();
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "field1");
+        config.put("ignore_missing", true);
+        String processorTag = randomAsciiOfLength(10);
+        LowercaseProcessor uppercaseProcessor = (LowercaseProcessor)factory.create(null, processorTag, config);
+        assertThat(uppercaseProcessor.getTag(), equalTo(processorTag));
+        assertThat(uppercaseProcessor.getField(), equalTo("field1"));
+        assertThat(uppercaseProcessor.isIgnoreMissing(), is(true));
     }
 
     public void testCreateMissingField() throws Exception {

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/LowercaseProcessorTests.java

@@ -23,8 +23,8 @@ import java.util.Locale;
 
 public class LowercaseProcessorTests extends AbstractStringProcessorTestCase {
     @Override
-    protected AbstractStringProcessor newProcessor(String field) {
-        return new LowercaseProcessor(randomAsciiOfLength(10), field);
+    protected AbstractStringProcessor newProcessor(String field, boolean ignoreMissing) {
+        return new LowercaseProcessor(randomAsciiOfLength(10), field, ignoreMissing);
     }
 
     @Override

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java

@@ -39,6 +39,21 @@ public class RenameProcessorFactoryTests extends ESTestCase {
         assertThat(renameProcessor.getTag(), equalTo(processorTag));
         assertThat(renameProcessor.getField(), equalTo("old_field"));
         assertThat(renameProcessor.getTargetField(), equalTo("new_field"));
+        assertThat(renameProcessor.isIgnoreMissing(), equalTo(false));
+    }
+
+    public void testCreateWithIgnoreMissing() throws Exception {
+        RenameProcessor.Factory factory = new RenameProcessor.Factory();
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "old_field");
+        config.put("target_field", "new_field");
+        config.put("ignore_missing", true);
+        String processorTag = randomAsciiOfLength(10);
+        RenameProcessor renameProcessor = factory.create(null, processorTag, config);
+        assertThat(renameProcessor.getTag(), equalTo(processorTag));
+        assertThat(renameProcessor.getField(), equalTo("old_field"));
+        assertThat(renameProcessor.getTargetField(), equalTo("new_field"));
+        assertThat(renameProcessor.isIgnoreMissing(), equalTo(true));
     }
 
     public void testCreateNoFieldPresent() throws Exception {

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java

@@ -30,6 +30,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.nullValue;
@@ -44,7 +45,7 @@ public class RenameProcessorTests extends ESTestCase {
         do {
             newFieldName = RandomDocumentPicks.randomFieldName(random());
         } while (RandomDocumentPicks.canAddField(newFieldName, ingestDocument) == false || newFieldName.equals(fieldName));
-        Processor processor = new RenameProcessor(randomAsciiOfLength(10), fieldName, newFieldName);
+        Processor processor = new RenameProcessor(randomAsciiOfLength(10), fieldName, newFieldName, false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(newFieldName, Object.class), equalTo(fieldValue));
     }
@@ -62,7 +63,7 @@ public class RenameProcessorTests extends ESTestCase {
         document.put("one", one);
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
 
-        Processor processor = new RenameProcessor(randomAsciiOfLength(10), "list.0", "item");
+        Processor processor = new RenameProcessor(randomAsciiOfLength(10), "list.0", "item", false);
         processor.execute(ingestDocument);
         Object actualObject = ingestDocument.getSourceAndMetadata().get("list");
         assertThat(actualObject, instanceOf(List.class));
@@ -75,7 +76,7 @@ public class RenameProcessorTests extends ESTestCase {
         assertThat(actualObject, instanceOf(String.class));
         assertThat(actualObject, equalTo("item1"));
 
-        processor = new RenameProcessor(randomAsciiOfLength(10), "list.0", "list.3");
+        processor = new RenameProcessor(randomAsciiOfLength(10), "list.0", "list.3", false);
         try {
             processor.execute(ingestDocument);
             fail("processor execute should have failed");
@@ -90,7 +91,8 @@ public class RenameProcessorTests extends ESTestCase {
     public void testRenameNonExistingField() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
         String fieldName = RandomDocumentPicks.randomFieldName(random());
-        Processor processor = new RenameProcessor(randomAsciiOfLength(10), fieldName, RandomDocumentPicks.randomFieldName(random()));
+        Processor processor = new RenameProcessor(randomAsciiOfLength(10), fieldName,
+            RandomDocumentPicks.randomFieldName(random()), false);
         try {
             processor.execute(ingestDocument);
             fail("processor execute should have failed");
@@ -99,11 +101,21 @@ public class RenameProcessorTests extends ESTestCase {
         }
     }
 
+    public void testRenameNonExistingFieldWithIgnoreMissing() throws Exception {
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        String fieldName = RandomDocumentPicks.randomFieldName(random());
+        Processor processor = new RenameProcessor(randomAsciiOfLength(10), fieldName,
+            RandomDocumentPicks.randomFieldName(random()), true);
+        processor.execute(ingestDocument);
+        assertIngestDocument(originalIngestDocument, ingestDocument);
+    }
+
     public void testRenameNewFieldAlreadyExists() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
         String fieldName = RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument);
         Processor processor = new RenameProcessor(randomAsciiOfLength(10), RandomDocumentPicks.randomExistingFieldName(
-                random(), ingestDocument), fieldName);
+                random(), ingestDocument), fieldName, false);
         try {
             processor.execute(ingestDocument);
             fail("processor execute should have failed");
@@ -117,7 +129,7 @@ public class RenameProcessorTests extends ESTestCase {
         String fieldName = RandomDocumentPicks.randomFieldName(random());
         ingestDocument.setFieldValue(fieldName, null);
         String newFieldName = RandomDocumentPicks.randomFieldName(random());
-        Processor processor = new RenameProcessor(randomAsciiOfLength(10), fieldName, newFieldName);
+        Processor processor = new RenameProcessor(randomAsciiOfLength(10), fieldName, newFieldName, false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.hasField(fieldName), equalTo(false));
         assertThat(ingestDocument.hasField(newFieldName), equalTo(true));
@@ -137,7 +149,7 @@ public class RenameProcessorTests extends ESTestCase {
         source.put("list", Collections.singletonList("item"));
 
         IngestDocument ingestDocument = new IngestDocument(source, Collections.emptyMap());
-        Processor processor = new RenameProcessor(randomAsciiOfLength(10), "list", "new_field");
+        Processor processor = new RenameProcessor(randomAsciiOfLength(10), "list", "new_field", false);
         try {
             processor.execute(ingestDocument);
             fail("processor execute should have failed");
@@ -161,7 +173,7 @@ public class RenameProcessorTests extends ESTestCase {
         source.put("list", Collections.singletonList("item"));
 
         IngestDocument ingestDocument = new IngestDocument(source, Collections.emptyMap());
-        Processor processor = new RenameProcessor(randomAsciiOfLength(10), "list", "new_field");
+        Processor processor = new RenameProcessor(randomAsciiOfLength(10), "list", "new_field", false);
         try {
             processor.execute(ingestDocument);
             fail("processor execute should have failed");
@@ -176,12 +188,12 @@ public class RenameProcessorTests extends ESTestCase {
         Map<String, Object> source = new HashMap<>();
         source.put("foo", "bar");
         IngestDocument ingestDocument = new IngestDocument(source, Collections.emptyMap());
-        Processor processor1 = new RenameProcessor(randomAsciiOfLength(10), "foo", "foo.bar");
+        Processor processor1 = new RenameProcessor(randomAsciiOfLength(10), "foo", "foo.bar", false);
         processor1.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue("foo", Map.class), equalTo(Collections.singletonMap("bar", "bar")));
         assertThat(ingestDocument.getFieldValue("foo.bar", String.class), equalTo("bar"));
 
-        Processor processor2 = new RenameProcessor(randomAsciiOfLength(10), "foo.bar", "foo.bar.baz");
+        Processor processor2 = new RenameProcessor(randomAsciiOfLength(10), "foo.bar", "foo.bar.baz", false);
         processor2.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue("foo", Map.class), equalTo(Collections.singletonMap("bar",
                 Collections.singletonMap("baz", "bar"))));
@@ -189,7 +201,7 @@ public class RenameProcessorTests extends ESTestCase {
         assertThat(ingestDocument.getFieldValue("foo.bar.baz", String.class), equalTo("bar"));
 
         // for fun lets try to restore it (which don't allow today)
-        Processor processor3 = new RenameProcessor(randomAsciiOfLength(10), "foo.bar.baz", "foo");
+        Processor processor3 = new RenameProcessor(randomAsciiOfLength(10), "foo.bar.baz", "foo", false);
         Exception e = expectThrows(IllegalArgumentException.class, () -> processor3.execute(ingestDocument));
         assertThat(e.getMessage(), equalTo("field [foo] already exists"));
     }

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TrimProcessorFactoryTests.java

@@ -26,6 +26,7 @@ import java.util.HashMap;
 import java.util.Map;
 
 import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.is;
 
 public class TrimProcessorFactoryTests extends ESTestCase {
 
@@ -37,6 +38,19 @@ public class TrimProcessorFactoryTests extends ESTestCase {
         TrimProcessor uppercaseProcessor = (TrimProcessor)factory.create(null, processorTag, config);
         assertThat(uppercaseProcessor.getTag(), equalTo(processorTag));
         assertThat(uppercaseProcessor.getField(), equalTo("field1"));
+        assertThat(uppercaseProcessor.isIgnoreMissing(), is(false));
+    }
+
+    public void testCreateWithIgnoreMissing() throws Exception {
+        TrimProcessor.Factory factory = new TrimProcessor.Factory();
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "field1");
+        config.put("ignore_missing", true);
+        String processorTag = randomAsciiOfLength(10);
+        TrimProcessor uppercaseProcessor = (TrimProcessor)factory.create(null, processorTag, config);
+        assertThat(uppercaseProcessor.getTag(), equalTo(processorTag));
+        assertThat(uppercaseProcessor.getField(), equalTo("field1"));
+        assertThat(uppercaseProcessor.isIgnoreMissing(), is(true));
     }
 
     public void testCreateMissingField() throws Exception {

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TrimProcessorTests.java

@@ -22,8 +22,8 @@ package org.elasticsearch.ingest.common;
 public class TrimProcessorTests extends AbstractStringProcessorTestCase {
 
     @Override
-    protected AbstractStringProcessor newProcessor(String field) {
-        return new TrimProcessor(randomAsciiOfLength(10), field);
+    protected AbstractStringProcessor newProcessor(String field, boolean ignoreMissing) {
+        return new TrimProcessor(randomAsciiOfLength(10), field, ignoreMissing);
     }
 
     @Override

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UppercaseProcessorTests.java

@@ -24,8 +24,8 @@ import java.util.Locale;
 public class UppercaseProcessorTests extends AbstractStringProcessorTestCase {
 
     @Override
-    protected AbstractStringProcessor newProcessor(String field) {
-        return new UppercaseProcessor(randomAsciiOfLength(10), field);
+    protected AbstractStringProcessor newProcessor(String field, boolean ignoreMissing) {
+        return new UppercaseProcessor(randomAsciiOfLength(10), field, ignoreMissing);
     }
 
     @Override

test/framework/src/main/java/org/elasticsearch/ingest/IngestDocumentMatcher.java

@@ -0,0 +1,67 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest;
+
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertThat;
+
+public class IngestDocumentMatcher {
+    /**
+     * Helper method to assert the equivalence between two IngestDocuments.
+     *
+     * @param a first object to compare
+     * @param b second object to compare
+     */
+    public static void assertIngestDocument(Object a, Object b) {
+        if (a instanceof Map) {
+            Map<?, ?> mapA = (Map<?, ?>) a;
+            Map<?, ?> mapB = (Map<?, ?>) b;
+            for (Map.Entry<?, ?> entry : mapA.entrySet()) {
+                if (entry.getValue() instanceof List || entry.getValue() instanceof Map) {
+                    assertIngestDocument(entry.getValue(), mapB.get(entry.getKey()));
+                }
+            }
+        } else if (a instanceof List) {
+            List<?> listA = (List<?>) a;
+            List<?> listB = (List<?>) b;
+            for (int i = 0; i < listA.size(); i++) {
+                Object value = listA.get(i);
+                if (value instanceof List || value instanceof Map) {
+                    assertIngestDocument(value, listB.get(i));
+                }
+            }
+        } else if (a instanceof byte[]) {
+            assertArrayEquals((byte[]) a, (byte[])b);
+        } else if (a instanceof IngestDocument) {
+            IngestDocument docA = (IngestDocument) a;
+            IngestDocument docB = (IngestDocument) b;
+            assertIngestDocument(docA.getSourceAndMetadata(), docB.getSourceAndMetadata());
+            assertIngestDocument(docA.getIngestMetadata(), docB.getIngestMetadata());
+        } else {
+            String msg = String.format(Locale.ROOT, "Expected %s class to be equal to %s", a.getClass().getName(), b.getClass().getName());
+            assertThat(msg, a, equalTo(b));
+        }
+    }
+}