Allow metadata fields in the _source (#61590)

* Configurable metadata field mappers in the _source

* Changes to support metadata fields in _source
Added test testDocumentContainsAllowedMetadataField()

* Merged DocumentParserTests from master
Fixed broken tests

* Handle non string values

* Allow metadata fields to parse values/objects/arrays/null

* Removed MetadataFieldMapper.isAllowedInSource() method

Delegated this functionality to MetadataFieldMapper.parse()

* Fixed bug that caused tests to break

* Cleanup parsing for existing metadata fields

* Cleanup parsing for existing metadata fields

* Remove doParse() method

* Fix broken test

* Lookup metadata mapper by name

Instead of linear scan

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureMetaFieldMapper.java

@@ -65,17 +65,6 @@ public class RankFeatureMetaFieldMapper extends MetadataFieldMapper {
         super(RankFeatureMetaFieldType.INSTANCE);
     }
 
-    @Override
-    public void preParse(ParseContext context) {}
-
-    @Override
-    protected void parseCreateField(ParseContext context) {
-        throw new AssertionError("Should never be called");
-    }
-
-    @Override
-    public void postParse(ParseContext context) {}
-
     @Override
     protected String contentType() {
         return CONTENT_TYPE;

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureMetaFieldMapperTests.java

@@ -69,6 +69,7 @@ public class RankFeatureMetaFieldMapperTests extends ESSingleNodeTestCase {
         BytesReference bytes = BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field(rfMetaField, 0).endObject());
         MapperParsingException e = expectThrows(MapperParsingException.class, () ->
             mapper.parse(new SourceToParse("test", "1", bytes, XContentType.JSON)));
-        assertTrue(e.getMessage().contains("Field ["+ rfMetaField + "] is a metadata field and cannot be added inside a document."));
+        assertTrue(
+            e.getCause().getMessage().contains("Field ["+ rfMetaField + "] is a metadata field and cannot be added inside a document."));
     }
 }

plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java

@@ -79,23 +79,9 @@ public class SizeFieldMapper extends MetadataFieldMapper {
         return this.enabled.value();
     }
 
-    @Override
-    public void preParse(ParseContext context) {
-    }
-
     @Override
     public void postParse(ParseContext context) throws IOException {
         // we post parse it so we get the size stored, possibly compressed (source will be preParse)
-        super.parse(context);
-    }
-
-    @Override
-    public void parse(ParseContext context) {
-        // nothing to do here, we call the parent in postParse
-    }
-
-    @Override
-    protected void parseCreateField(ParseContext context) {
         if (enabled.value() == false) {
             return;
         }

server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java

@@ -392,10 +392,7 @@ final class DocumentParser {
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
                 paths = splitAndValidatePath(currentFieldName);
-                if (context.mapperService().isMetadataField(context.path().pathAsText(currentFieldName))) {
-                    throw new MapperParsingException("Field [" + currentFieldName + "] is a metadata field and cannot be added inside"
-                        + " a document. Use the index API request parameters.");
-                } else if (containsDisabledObjectMapper(mapper, paths)) {
+                if (containsDisabledObjectMapper(mapper, paths)) {
                     parser.nextToken();
                     parser.skipChildren();
                 }
@@ -483,7 +480,7 @@ final class DocumentParser {
                                     String[] paths) throws IOException {
         assert currentFieldName != null;
 
-        Mapper objectMapper = getMapper(mapper, currentFieldName, paths);
+        Mapper objectMapper = getMapper(context, mapper, currentFieldName, paths);
         if (objectMapper != null) {
             context.path().add(currentFieldName);
             parseObjectOrField(context, objectMapper);
@@ -520,7 +517,7 @@ final class DocumentParser {
                                    String[] paths) throws IOException {
         String arrayFieldName = lastFieldName;
 
-        Mapper mapper = getMapper(parentMapper, lastFieldName, paths);
+        Mapper mapper = getMapper(context, parentMapper, lastFieldName, paths);
         if (mapper != null) {
             // There is a concrete mapper for this field already. Need to check if the mapper
             // expects an array, if so we pass the context straight to the mapper and if not
@@ -597,7 +594,7 @@ final class DocumentParser {
             throw new MapperParsingException("object mapping [" + parentMapper.name() + "] trying to serialize a value with"
                 + " no field associated with it, current value [" + context.parser().textOrNull() + "]");
         }
-        Mapper mapper = getMapper(parentMapper, currentFieldName, paths);
+        Mapper mapper = getMapper(context, parentMapper, currentFieldName, paths);
         if (mapper != null) {
             parseObjectOrField(context, mapper);
         } else {
@@ -614,7 +611,7 @@ final class DocumentParser {
     private static void parseNullValue(ParseContext context, ObjectMapper parentMapper, String lastFieldName,
                                        String[] paths) throws IOException {
         // we can only handle null values if we have mappings for them
-        Mapper mapper = getMapper(parentMapper, lastFieldName, paths);
+        Mapper mapper = getMapper(context, parentMapper, lastFieldName, paths);
         if (mapper != null) {
             // TODO: passing null to an object seems bogus?
             parseObjectOrField(context, mapper);
@@ -882,9 +879,16 @@ final class DocumentParser {
     }
 
     // looks up a child mapper, but takes into account field names that expand to objects
-    private static Mapper getMapper(ObjectMapper objectMapper, String fieldName, String[] subfields) {
+    private static Mapper getMapper(final ParseContext context, ObjectMapper objectMapper, String fieldName, String[] subfields) {
+        String fieldPath = context.path().pathAsText(fieldName);
+        // Check if mapper is a metadata mapper first
+        Mapper mapper = context.docMapper().mapping().getMetadataMapper(fieldPath);
+        if (mapper != null) {
+            return mapper;
+        }
+
         for (int i = 0; i < subfields.length - 1; ++i) {
-            Mapper mapper = objectMapper.getMapper(subfields[i]);
+            mapper = objectMapper.getMapper(subfields[i]);
             if (mapper == null || (mapper instanceof ObjectMapper) == false) {
                 return null;
             }

server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java

@@ -27,7 +27,6 @@ import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.index.query.QueryShardContext;
 
-import java.io.IOException;
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
@@ -160,19 +159,6 @@ public class FieldNamesFieldMapper extends MetadataFieldMapper {
         return (FieldNamesFieldType) super.fieldType();
     }
 
-    @Override
-    public void preParse(ParseContext context) {
-    }
-
-    @Override
-    public void postParse(ParseContext context) {
-    }
-
-    @Override
-    protected void parseCreateField(ParseContext context) throws IOException {
-        // Adding values to the _field_names field is handled by the mappers for each field type
-    }
-
     static Iterable<String> extractFieldNames(final String fullPath) {
         return new Iterable<String>() {
             @Override

server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java

@@ -257,11 +257,6 @@ public class IdFieldMapper extends MetadataFieldMapper {
 
     @Override
     public void preParse(ParseContext context) throws IOException {
-        super.parse(context);
-    }
-
-    @Override
-    protected void parseCreateField(ParseContext context) throws IOException {
         BytesRef id = Uid.encodeId(context.sourceToParse().id());
         context.doc().add(new Field(NAME, id, Defaults.FIELD_TYPE));
     }

server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java

@@ -82,22 +82,8 @@ public final class IgnoredFieldMapper extends MetadataFieldMapper {
         super(IgnoredFieldType.INSTANCE);
     }
 
-    @Override
-    public void preParse(ParseContext context) throws IOException {
-    }
-
     @Override
     public void postParse(ParseContext context) throws IOException {
-        super.parse(context);
-    }
-
-    @Override
-    public void parse(ParseContext context) throws IOException {
-        // done in post-parse
-    }
-
-    @Override
-    protected void parseCreateField(ParseContext context) throws IOException {
         for (String field : context.getIgnoredFields()) {
             context.doc().add(new Field(NAME, field, Defaults.FIELD_TYPE));
         }

server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java

@@ -27,7 +27,6 @@ import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
 import org.elasticsearch.search.lookup.SearchLookup;
 
-import java.io.IOException;
 import java.util.Collections;
 import java.util.function.Supplier;
 
@@ -73,12 +72,6 @@ public class IndexFieldMapper extends MetadataFieldMapper {
         super(IndexFieldType.INSTANCE);
     }
 
-    @Override
-    public void preParse(ParseContext context) throws IOException {}
-
-    @Override
-    protected void parseCreateField(ParseContext context) throws IOException {}
-
     @Override
     protected String contentType() {
         return CONTENT_TYPE;

server/src/main/java/org/elasticsearch/index/mapper/Mapping.java

@@ -48,6 +48,7 @@ public final class Mapping implements ToXContentFragment {
     final RootObjectMapper root;
     final MetadataFieldMapper[] metadataMappers;
     final Map<Class<? extends MetadataFieldMapper>, MetadataFieldMapper> metadataMappersMap;
+    final Map<String, MetadataFieldMapper> metadataMappersByName;
     final Map<String, Object> meta;
 
     public Mapping(Version indexCreated, RootObjectMapper rootObjectMapper,
@@ -55,8 +56,10 @@ public final class Mapping implements ToXContentFragment {
         this.indexCreated = indexCreated;
         this.metadataMappers = metadataMappers;
         Map<Class<? extends MetadataFieldMapper>, MetadataFieldMapper> metadataMappersMap = new HashMap<>();
+        Map<String, MetadataFieldMapper> metadataMappersByName = new HashMap<>();
         for (MetadataFieldMapper metadataMapper : metadataMappers) {
             metadataMappersMap.put(metadataMapper.getClass(), metadataMapper);
+            metadataMappersByName.put(metadataMapper.name(), metadataMapper);
         }
         this.root = rootObjectMapper;
         // keep root mappers sorted for consistent serialization
@@ -67,6 +70,7 @@ public final class Mapping implements ToXContentFragment {
             }
         });
         this.metadataMappersMap = unmodifiableMap(metadataMappersMap);
+        this.metadataMappersByName = unmodifiableMap(metadataMappersByName);
         this.meta = meta;
     }
 
@@ -136,6 +140,10 @@ public final class Mapping implements ToXContentFragment {
         return new Mapping(indexCreated, mergedRoot, mergedMetadataMappers.values().toArray(new MetadataFieldMapper[0]), mergedMeta);
     }
 
+    public MetadataFieldMapper getMetadataMapper(String mapperName) {
+        return metadataMappersByName.get(mapperName);
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         root.toXContent(builder, params, new ToXContent() {

server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java

@@ -153,10 +153,18 @@ public abstract class MetadataFieldMapper extends ParametrizedFieldMapper {
         return builder.endObject();
     }
 
+    @Override
+    protected void parseCreateField(ParseContext context) throws IOException {
+        throw new MapperParsingException("Field [" + name() + "] is a metadata field and cannot be added inside"
+            + " a document. Use the index API request parameters.");
+    }
+
     /**
      * Called before {@link FieldMapper#parse(ParseContext)} on the {@link RootObjectMapper}.
      */
-    public abstract void preParse(ParseContext context) throws IOException;
+    public void preParse(ParseContext context) throws IOException {
+        // do nothing
+    }
 
     /**
      * Called after {@link FieldMapper#parse(ParseContext)} on the {@link RootObjectMapper}.
@@ -169,5 +177,4 @@ public abstract class MetadataFieldMapper extends ParametrizedFieldMapper {
     public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup lookup, String format) {
         throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "].");
     }
-
 }

server/src/main/java/org/elasticsearch/index/mapper/NestedPathFieldMapper.java

@@ -31,7 +31,6 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.query.QueryShardContext;
 
-import java.io.IOException;
 import java.util.Collections;
 
 public class NestedPathFieldMapper extends MetadataFieldMapper {
@@ -93,24 +92,9 @@ public class NestedPathFieldMapper extends MetadataFieldMapper {
         super(new NestedPathFieldType(settings));
     }
 
-    @Override
-    public void preParse(ParseContext context) throws IOException {
-        super.parse(context);
-    }
-
-    @Override
-    public void parse(ParseContext context) throws IOException {
-        // we parse in pre parse
-    }
-
-    @Override
-    protected void parseCreateField(ParseContext context) throws IOException {
-    }
-
     @Override
     protected String contentType() {
         return NAME;
     }
 
-
 }

server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java

@@ -117,18 +117,6 @@ public class RoutingFieldMapper extends MetadataFieldMapper {
 
     @Override
     public void preParse(ParseContext context) throws IOException {
-        super.parse(context);
-    }
-
-    @Override
-    public void parse(ParseContext context) throws IOException {
-        // no need ot parse here, we either get the routing in the sourceToParse
-        // or we don't have routing, if we get it in sourceToParse, we process it in preParse
-        // which will always be called
-    }
-
-    @Override
-    protected void parseCreateField(ParseContext context) throws IOException {
         String routing = context.sourceToParse().routing();
         if (routing != null) {
             context.doc().add(new Field(fieldType().name(), routing, Defaults.FIELD_TYPE));

server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java

@@ -182,11 +182,6 @@ public class SeqNoFieldMapper extends MetadataFieldMapper {
 
     @Override
     public void preParse(ParseContext context) throws IOException {
-        super.parse(context);
-    }
-
-    @Override
-    protected void parseCreateField(ParseContext context) throws IOException {
         // see InternalEngine.innerIndex to see where the real version value is set
         // also see ParsedDocument.updateSeqID (called by innerIndex)
         SequenceIDFields seqID = SequenceIDFields.emptySeqID();
@@ -196,11 +191,6 @@ public class SeqNoFieldMapper extends MetadataFieldMapper {
         context.doc().add(seqID.primaryTerm);
     }
 
-    @Override
-    public void parse(ParseContext context) throws IOException {
-        // fields are added in parseCreateField
-    }
-
     @Override
     public void postParse(ParseContext context) throws IOException {
         // In the case of nested docs, let's fill nested docs with the original

server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java

@@ -156,16 +156,6 @@ public class SourceFieldMapper extends MetadataFieldMapper {
 
     @Override
     public void preParse(ParseContext context) throws IOException {
-        super.parse(context);
-    }
-
-    @Override
-    public void parse(ParseContext context) throws IOException {
-        // nothing to do here, we will call it in pre parse
-    }
-
-    @Override
-    protected void parseCreateField(ParseContext context) throws IOException {
         BytesReference originalSource = context.sourceToParse().source();
         XContentType contentType = context.sourceToParse().getXContentType();
         final BytesReference adaptedSource = applyFilters(originalSource, contentType);

server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java

@@ -168,21 +168,6 @@ public class TypeFieldMapper extends MetadataFieldMapper {
         super(new TypeFieldType());
     }
 
-    @Override
-    public void preParse(ParseContext context) throws IOException {
-        super.parse(context);
-    }
-
-    @Override
-    public void parse(ParseContext context) throws IOException {
-        // we parse in pre parse
-    }
-
-    @Override
-    protected void parseCreateField(ParseContext context) throws IOException {
-
-    }
-
     @Override
     protected String contentType() {
         return CONTENT_TYPE;

server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java

@@ -68,22 +68,12 @@ public class VersionFieldMapper extends MetadataFieldMapper {
 
     @Override
     public void preParse(ParseContext context) throws IOException {
-        super.parse(context);
-    }
-
-    @Override
-    protected void parseCreateField(ParseContext context) throws IOException {
         // see InternalEngine.updateVersion to see where the real version value is set
         final Field version = new NumericDocValuesField(NAME, -1L);
         context.version(version);
         context.doc().add(version);
     }
 
-    @Override
-    public void parse(ParseContext context) throws IOException {
-        // _version added in preparse
-    }
-
     @Override
     public void postParse(ParseContext context) throws IOException {
         // In the case of nested docs, let's fill nested docs with version=1 so that Lucene doesn't write a Bitset for documents

server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java

@@ -47,7 +47,6 @@ import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
 import org.elasticsearch.index.mapper.ParametrizedFieldMapper;
-import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.TextSearchInfo;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.indices.IndexTemplateMissingException;
@@ -1554,16 +1553,6 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
             return new MetadataTimestampFieldBuilder().init(this);
         }
 
-        @Override
-        public void preParse(ParseContext context) {
-
-        }
-
-        @Override
-        protected void parseCreateField(ParseContext context) {
-
-        }
-
         @Override
         protected String contentType() {
             return "_data_stream_timestamp";

server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java

@@ -31,15 +31,18 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.ParseContext.Document;
+import org.elasticsearch.plugins.Plugin;
 
 import java.io.IOException;
 import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 
+import static java.util.Collections.singletonList;
 import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath;
 import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
 import static org.hamcrest.Matchers.containsString;
@@ -50,6 +53,11 @@ import static org.hamcrest.Matchers.notNullValue;
 
 public class DocumentParserTests extends MapperServiceTestCase {
 
+    @Override
+    protected Collection<? extends Plugin> getPlugins() {
+        return singletonList(new MockMetadataMapperPlugin());
+    }
+
     public void testFieldDisabled() throws Exception {
         DocumentMapper mapper = createDocumentMapper(mapping(b -> {
             b.startObject("foo").field("enabled", false).endObject();
@@ -931,12 +939,36 @@ public class DocumentParserTests extends MapperServiceTestCase {
         DocumentMapper mapper = createDocumentMapper(mapping(b -> {}));
         MapperParsingException e = expectThrows(MapperParsingException.class, () ->
             mapper.parse(source(b -> b.field("_field_names", 0))));
-        assertTrue(e.getMessage(),
-            e.getMessage().contains("Field [_field_names] is a metadata field and cannot be added inside a document."));
+        assertTrue(e.getCause().getMessage(),
+            e.getCause().getMessage().contains("Field [_field_names] is a metadata field and cannot be added inside a document."));
 
         mapper.parse(source(b -> b.field("foo._field_names", 0))); // parses without error
     }
 
+    public void testDocumentContainsAllowedMetadataField() throws Exception {
+        DocumentMapper mapper = createDocumentMapper(mapping(b -> {}));
+        {
+            // A metadata field that parses a value fails to parse a null value
+            MapperParsingException e = expectThrows(MapperParsingException.class, () ->
+                mapper.parse(source(b -> b.nullField(MockMetadataMapperPlugin.MockMetadataMapper.CONTENT_TYPE))));
+            assertTrue(e.getMessage(), e.getMessage().contains("failed to parse field [_mock_metadata]"));
+        }
+        {
+            // A metadata field that parses a value fails to parse an object
+            MapperParsingException e = expectThrows(MapperParsingException.class, () ->
+                mapper.parse(source(b -> b.field(MockMetadataMapperPlugin.MockMetadataMapper.CONTENT_TYPE)
+                    .startObject().field("sub-field", "true").endObject())));
+            assertTrue(e.getMessage(), e.getMessage().contains("failed to parse field [_mock_metadata]"));
+        }
+        {
+            ParsedDocument doc = mapper.parse(source(b ->
+                b.field(MockMetadataMapperPlugin.MockMetadataMapper.CONTENT_TYPE, "mock-metadata-field-value")
+            ));
+            IndexableField field = doc.rootDoc().getField(MockMetadataMapperPlugin.MockMetadataMapper.CONTENT_TYPE);
+            assertEquals("mock-metadata-field-value", field.stringValue());
+        }
+    }
+
     public void testSimpleMapper() throws Exception {
         DocumentMapper docMapper = createDocumentMapper(mapping(b -> {
             b.startObject("name");

server/src/test/java/org/elasticsearch/index/mapper/ExternalMetadataMapper.java

@@ -37,11 +37,6 @@ public class ExternalMetadataMapper extends MetadataFieldMapper {
         super(new BooleanFieldMapper.BooleanFieldType(FIELD_NAME));
     }
 
-    @Override
-    protected void parseCreateField(ParseContext context) throws IOException {
-        // handled in post parse
-    }
-
     @Override
     public Iterator<Mapper> iterator() {
         return Collections.emptyIterator();
@@ -52,10 +47,6 @@ public class ExternalMetadataMapper extends MetadataFieldMapper {
         return CONTENT_TYPE;
     }
 
-    @Override
-    public void preParse(ParseContext context) throws IOException {
-    }
-
     @Override
     public void postParse(ParseContext context) throws IOException {
         context.doc().add(new StringField(FIELD_NAME, FIELD_VALUE, Store.YES));

server/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java

@@ -59,7 +59,8 @@ public class IdFieldMapperTests extends ESSingleNodeTestCase {
                 .startObject().field("_id", "1").endObject()), XContentType.JSON));
             fail("Expected failure to parse metadata field");
         } catch (MapperParsingException e) {
-            assertTrue(e.getMessage(), e.getMessage().contains("Field [_id] is a metadata field and cannot be added inside a document"));
+            assertTrue(e.getCause().getMessage(),
+                e.getCause().getMessage().contains("Field [_id] is a metadata field and cannot be added inside a document"));
         }
     }
 

server/src/test/java/org/elasticsearch/index/mapper/MockMetadataMapperPlugin.java

@@ -0,0 +1,98 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.plugins.MapperPlugin;
+import org.elasticsearch.plugins.Plugin;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Mapper plugin providing a mock metadata field mapper implementation that supports setting its value
+ * through the document source.
+ */
+public class MockMetadataMapperPlugin extends Plugin implements MapperPlugin {
+
+    /**
+     * A mock metadata field mapper that supports being set from the document source.
+     */
+    public static class MockMetadataMapper extends MetadataFieldMapper {
+
+        static final String CONTENT_TYPE = "_mock_metadata";
+        static final String FIELD_NAME = "_mock_metadata";
+
+        protected MockMetadataMapper() {
+            super(new KeywordFieldMapper.KeywordFieldType(FIELD_NAME));
+        }
+
+        @Override
+        protected void parseCreateField(ParseContext context) throws IOException {
+            if (context.parser().currentToken() == XContentParser.Token.VALUE_STRING) {
+                context.doc().add(new StringField(FIELD_NAME, context.parser().text(), Field.Store.YES));
+            } else {
+                throw new IllegalArgumentException("Field [" + fieldType().name() + "] must be a string.");
+            }
+        }
+
+        @Override
+        public Iterator<Mapper> iterator() {
+            return Collections.emptyIterator();
+        }
+
+        @Override
+        protected String contentType() {
+            return CONTENT_TYPE;
+        }
+
+        public static class Builder extends MetadataFieldMapper.Builder {
+
+            protected Builder() {
+                super(FIELD_NAME);
+            }
+
+            @Override
+            protected List<Parameter<?>> getParameters() {
+                return Collections.emptyList();
+            }
+
+            @Override
+            public MockMetadataMapper build(BuilderContext context) {
+                return new MockMetadataMapper();
+            }
+        }
+
+        public static final TypeParser PARSER = new ConfigurableTypeParser(
+            c -> new MockMetadataMapper(),
+            c -> new MockMetadataMapper.Builder()) {
+        };
+    }
+
+    @Override
+    public Map<String, MetadataFieldMapper.TypeParser> getMetadataMappers() {
+        return Collections.singletonMap(MockMetadataMapper.CONTENT_TYPE, MockMetadataMapper.PARSER);
+    }
+}

server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java

@@ -58,7 +58,7 @@ public class RoutingFieldMapperTests extends ESSingleNodeTestCase {
                 .startObject().field("_routing", "foo").endObject()),XContentType.JSON));
             fail("Expected failure to parse metadata field");
         } catch (MapperParsingException e) {
-            assertThat(e.getMessage(), e.getMessage(),
+            assertThat(e.getCause().getMessage(), e.getCause().getMessage(),
                 containsString("Field [_routing] is a metadata field and cannot be added inside a document"));
         }
     }

x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java

@@ -171,15 +171,6 @@ public class DataStreamTimestampFieldMapper extends MetadataFieldMapper {
         }
     }
 
-    @Override
-    public void preParse(ParseContext context) throws IOException {}
-
-    @Override
-    protected void parseCreateField(ParseContext context) throws IOException {
-        // Meta field doesn't create any fields, so this shouldn't happen.
-        throw new IllegalStateException(NAME + " field mapper cannot create fields");
-    }
-
     @Override
     public void postParse(ParseContext context) throws IOException {
         if (enabled == false) {