Merge pull request #15213 from brwe/copy-to-in-multi-fields-exception

throw exception if a copy_to is within a multi field

Copy to within multi field is ignored from 2.0 on, see #10802.
Instead of just ignoring it, we should throw an exception if this
is found in the mapping when a mapping is added. For already
existing indices we should at least log a warning.
We remove the copy_to in any case.

related to #14946

core/src/main/java/org/elasticsearch/index/mapper/Mapper.java

@@ -134,6 +134,26 @@ public abstract class Mapper implements ToXContent, Iterable<Mapper> {
             public ParseFieldMatcher parseFieldMatcher() {
                 return parseFieldMatcher;
             }
+
+            public boolean isWithinMultiField() { return false; }
+
+            protected Function<String, TypeParser> typeParsers() { return typeParsers; }
+
+            protected Function<String, SimilarityProvider> similarityLookupService() { return similarityLookupService; }
+
+            public ParserContext createMultiFieldContext(ParserContext in) {
+                return new MultiFieldParserContext(in) {
+                    @Override
+                    public boolean isWithinMultiField() { return true; }
+                };
+            }
+
+            class MultiFieldParserContext extends ParserContext {
+                MultiFieldParserContext(ParserContext in) {
+                    super(in.type(), in.analysisService, in.similarityLookupService(), in.mapperService(), in.typeParsers(), in.indexVersionCreated(), in.parseFieldMatcher());
+                }
+            }
+
         }
 
         Mapper.Builder<?,?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException;

core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java

@@ -25,6 +25,7 @@ import org.elasticsearch.Version;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.joda.FormatDateTimeFormatter;
 import org.elasticsearch.common.joda.Joda;
+import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.loader.SettingsLoader;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
@@ -184,11 +185,12 @@ public class TypeParsers {
     public static void parseField(FieldMapper.Builder builder, String name, Map<String, Object> fieldNode, Mapper.TypeParser.ParserContext parserContext) {
         NamedAnalyzer indexAnalyzer = builder.fieldType().indexAnalyzer();
         NamedAnalyzer searchAnalyzer = builder.fieldType().searchAnalyzer();
+        Version indexVersionCreated = parserContext.indexVersionCreated();
         for (Iterator<Map.Entry<String, Object>> iterator = fieldNode.entrySet().iterator(); iterator.hasNext();) {
             Map.Entry<String, Object> entry = iterator.next();
             final String propName = Strings.toUnderscoreCase(entry.getKey());
             final Object propNode = entry.getValue();
-            if (propName.equals("index_name") && parserContext.indexVersionCreated().before(Version.V_2_0_0_beta1)) {
+            if (propName.equals("index_name") && indexVersionCreated.before(Version.V_2_0_0_beta1)) {
                 builder.indexName(propNode.toString());
                 iterator.remove();
             } else if (propName.equals("store")) {
@@ -239,7 +241,7 @@ public class TypeParsers {
                 iterator.remove();
             } else if (propName.equals("omit_term_freq_and_positions")) {
                 final IndexOptions op = nodeBooleanValue(propNode) ? IndexOptions.DOCS : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
-                if (parserContext.indexVersionCreated().onOrAfter(Version.V_1_0_0_RC2)) {
+                if (indexVersionCreated.onOrAfter(Version.V_1_0_0_RC2)) {
                     throw new ElasticsearchParseException("'omit_term_freq_and_positions' is not supported anymore - use ['index_options' : 'docs']  instead");
                 }
                 // deprecated option for BW compat
@@ -249,8 +251,8 @@ public class TypeParsers {
                 builder.indexOptions(nodeIndexOptionValue(propNode));
                 iterator.remove();
             } else if (propName.equals("analyzer") || // for backcompat, reading old indexes, remove for v3.0
-                       propName.equals("index_analyzer") && parserContext.indexVersionCreated().before(Version.V_2_0_0_beta1)) {
-                
+                       propName.equals("index_analyzer") && indexVersionCreated.before(Version.V_2_0_0_beta1)) {
+
                 NamedAnalyzer analyzer = parserContext.analysisService().analyzer(propNode.toString());
                 if (analyzer == null) {
                     throw new MapperParsingException("analyzer [" + propNode.toString() + "] not found for field [" + name + "]");
@@ -267,10 +269,10 @@ public class TypeParsers {
             } else if (propName.equals("include_in_all")) {
                 builder.includeInAll(nodeBooleanValue(propNode));
                 iterator.remove();
-            } else if (propName.equals("postings_format") && parserContext.indexVersionCreated().before(Version.V_2_0_0_beta1)) {
+            } else if (propName.equals("postings_format") && indexVersionCreated.before(Version.V_2_0_0_beta1)) {
                 // ignore for old indexes
                 iterator.remove();
-            } else if (propName.equals("doc_values_format") && parserContext.indexVersionCreated().before(Version.V_2_0_0_beta1)) {
+            } else if (propName.equals("doc_values_format") && indexVersionCreated.before(Version.V_2_0_0_beta1)) {
                 // ignore for old indexes
                 iterator.remove();
             } else if (propName.equals("similarity")) {
@@ -281,7 +283,16 @@ public class TypeParsers {
                 builder.fieldDataSettings(settings);
                 iterator.remove();
             } else if (propName.equals("copy_to")) {
-                parseCopyFields(propNode, builder);
+                if (parserContext.isWithinMultiField()) {
+                    if (indexVersionCreated.after(Version.V_2_1_0) ||
+                        (indexVersionCreated.after(Version.V_2_0_1) && indexVersionCreated.before(Version.V_2_1_0))) {
+                        throw new MapperParsingException("copy_to in multi fields is not allowed. Found the copy_to in field [" + name + "] which is within a multi field.");
+                    } else {
+                        ESLoggerFactory.getLogger("mapping [" + parserContext.type() + "]").warn("Found a copy_to in field [" + name + "] which is within a multi field. This feature has been removed and the copy_to will be removed from the mapping.");
+                    }
+                } else {
+                    parseCopyFields(propNode, builder);
+                }
                 iterator.remove();
             }
         }
@@ -298,6 +309,7 @@ public class TypeParsers {
     }
 
     public static boolean parseMultiField(FieldMapper.Builder builder, String name, Mapper.TypeParser.ParserContext parserContext, String propName, Object propNode) {
+        parserContext = parserContext.createMultiFieldContext(parserContext);
         if (propName.equals("path") && parserContext.indexVersionCreated().before(Version.V_2_0_0_beta1)) {
             builder.multiFieldPathType(parsePathType(name, propNode.toString()));
             return true;

core/src/test/java/org/elasticsearch/index/mapper/core/MultiFieldCopyToMapperTests.java

@@ -0,0 +1,105 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+package org.elasticsearch.index.mapper.core;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.MapperTestUtils;
+import org.elasticsearch.index.mapper.DocumentMapper;
+import org.elasticsearch.index.mapper.MapperParsingException;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.VersionUtils;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.hamcrest.core.IsEqual.equalTo;
+
+public class MultiFieldCopyToMapperTests extends ESTestCase {
+
+    public void testExceptionForCopyToInMultiFields() throws IOException {
+        XContentBuilder mapping = createMappinmgWithCopyToInMultiField();
+        Tuple<List<Version>, List<Version>> versionsWithAndWithoutExpectedExceptions = versionsWithAndWithoutExpectedExceptions();
+
+        // first check that for newer versions we throw exception if copy_to is found withing multi field
+        Version indexVersion = randomFrom(versionsWithAndWithoutExpectedExceptions.v1());
+        MapperService mapperService = MapperTestUtils.newMapperService(createTempDir(), Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, indexVersion).build());
+        try {
+            mapperService.parse("type", new CompressedXContent(mapping.string()), true);
+            fail("Parsing should throw an exception because the mapping contains a copy_to in a multi field");
+        } catch (MapperParsingException e) {
+            assertThat(e.getMessage(), equalTo("copy_to in multi fields is not allowed. Found the copy_to in field [c] which is within a multi field."));
+        }
+
+        // now test that with an older version the pasring just works
+        indexVersion = randomFrom(versionsWithAndWithoutExpectedExceptions.v2());
+        mapperService = MapperTestUtils.newMapperService(createTempDir(), Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, indexVersion).build());
+        DocumentMapper documentMapper = mapperService.parse("type", new CompressedXContent(mapping.string()), true);
+        assertFalse(documentMapper.mapping().toString().contains("copy_to"));
+    }
+
+    private static XContentBuilder createMappinmgWithCopyToInMultiField() throws IOException {
+        XContentBuilder mapping = jsonBuilder();
+        mapping.startObject()
+            .startObject("type")
+            .startObject("properties")
+            .startObject("a")
+            .field("type", "string")
+            .endObject()
+            .startObject("b")
+            .field("type", "string")
+            .startObject("fields")
+            .startObject("c")
+            .field("type", "string")
+            .field("copy_to", "a")
+            .endObject()
+            .endObject()
+            .endObject()
+            .endObject()
+            .endObject()
+            .endObject();
+        return mapping;
+    }
+
+    // returs a tuple where
+    // v1 is a list of versions for which we expect an excpetion when a copy_to in multi fields is found and
+    // v2 is older versions where we throw no exception and we just log a warning
+    private static Tuple<List<Version>, List<Version>> versionsWithAndWithoutExpectedExceptions() {
+        List<Version> versionsWithException = new ArrayList<>();
+        List<Version> versionsWithoutException = new ArrayList<>();
+        for (Version version : VersionUtils.allVersions()) {
+            if (version.after(Version.V_2_1_0) ||
+                (version.after(Version.V_2_0_1) && version.before(Version.V_2_1_0))) {
+                versionsWithException.add(version);
+            } else {
+                versionsWithoutException.add(version);
+            }
+        }
+        return new Tuple<>(versionsWithException, versionsWithoutException);
+    }
+}

docs/reference/migration/migrate_2_0/mapping.asciidoc

@@ -429,3 +429,11 @@ to use the old default of 0. This was done to prevent phrase queries from
 matching across different values of the same term unexpectedly. Specifically,
 100 was chosen to cause phrase queries with slops up to 99 to match only within
 a single value of a field.
+
+==== copy_to and multi fields
+
+A <<copy-to,copy_to>> within a <<multi-fields,multi field>> is ignored from version 2.0 on. With any version after
+2.1 or 2.0.1 creating a mapping that has a copy_to within a multi field will result 
+in an exception.
+
+

plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/AttachmentUnitTestCase.java

@@ -22,13 +22,20 @@ package org.elasticsearch.mapper.attachments;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
 public class AttachmentUnitTestCase extends ESTestCase {
 
     protected Settings testSettings;
-    
+
+    protected static IndicesModule getIndicesModuleWithRegisteredAttachmentMapper() {
+        IndicesModule indicesModule = new IndicesModule();
+        indicesModule.registerMapper(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser());
+        return indicesModule;
+    }
+
     @Before
     public void createSettings() throws Exception {
       testSettings = Settings.builder()

plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/DateAttachmentMapperTests.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.mapper.attachments;
 
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.MapperTestUtils;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.DocumentMapperParser;
 import org.elasticsearch.index.mapper.core.StringFieldMapper;
@@ -37,7 +38,7 @@ public class DateAttachmentMapperTests extends AttachmentUnitTestCase {
 
     @Before
     public void setupMapperParser() throws Exception {
-        mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY).documentMapperParser();
+        mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser();
     }
 
     public void testSimpleMappings() throws Exception {

plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/EncryptedDocMapperTests.java

@@ -21,11 +21,11 @@ package org.elasticsearch.mapper.attachments;
 
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.MapperTestUtils;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.DocumentMapperParser;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.ParseContext;
-import org.elasticsearch.mapper.attachments.AttachmentMapper;
 
 import java.io.IOException;
 
@@ -42,7 +42,7 @@ import static org.hamcrest.Matchers.*;
 public class EncryptedDocMapperTests extends AttachmentUnitTestCase {
 
     public void testMultipleDocsEncryptedLast() throws IOException {
-        DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY).documentMapperParser();
+        DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser();
 
         String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/encrypted/test-mapping.json");
         DocumentMapper docMapper = mapperParser.parse(mapping);
@@ -72,7 +72,7 @@ public class EncryptedDocMapperTests extends AttachmentUnitTestCase {
     }
 
     public void testMultipleDocsEncryptedFirst() throws IOException {
-        DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY).documentMapperParser();
+        DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser();
         String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/encrypted/test-mapping.json");
         DocumentMapper docMapper = mapperParser.parse(mapping);
         byte[] html = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/htmlWithValidDateMeta.html");
@@ -103,9 +103,8 @@ public class EncryptedDocMapperTests extends AttachmentUnitTestCase {
     public void testMultipleDocsEncryptedNotIgnoringErrors() throws IOException {
         try {
             DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(),
-                Settings.builder()
-                    .put("index.mapping.attachment.ignore_errors", false)
-                    .build()).documentMapperParser();
+                Settings.builder().put("index.mapping.attachment.ignore_errors", false).build(),
+                getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser();
 
             String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/encrypted/test-mapping.json");
             DocumentMapper docMapper = mapperParser.parse(mapping);

plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/LanguageDetectionAttachmentMapperTests.java

@@ -21,11 +21,11 @@ package org.elasticsearch.mapper.attachments;
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.MapperTestUtils;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.DocumentMapperParser;
 import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.core.StringFieldMapper;
-import org.elasticsearch.mapper.attachments.AttachmentMapper;
 import org.junit.Before;
 
 import java.io.IOException;
@@ -50,9 +50,8 @@ public class LanguageDetectionAttachmentMapperTests extends AttachmentUnitTestCa
 
     public void setupMapperParser(boolean langDetect) throws IOException {
         DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(),
-            Settings.settingsBuilder()
-                .put("index.mapping.attachment.detect_language", langDetect)
-            .build()).documentMapperParser();
+            Settings.settingsBuilder().put("index.mapping.attachment.detect_language", langDetect).build(),
+            getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser();
         String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/language/language-mapping.json");
         docMapper = mapperParser.parse(mapping);
 

plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MetadataMapperTests.java

@@ -21,11 +21,11 @@ package org.elasticsearch.mapper.attachments;
 
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.MapperTestUtils;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.DocumentMapperParser;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.ParseContext;
-import org.elasticsearch.mapper.attachments.AttachmentMapper;
 
 import java.io.IOException;
 
@@ -44,7 +44,7 @@ public class MetadataMapperTests extends AttachmentUnitTestCase {
                                              .put(this.testSettings)
                                              .put(otherSettings)
                                              .build();
-        DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), settings).documentMapperParser();
+        DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), settings, getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser();
 
         String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/metadata/test-mapping.json");
         DocumentMapper docMapper = mapperParser.parse(mapping);

plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MultifieldAttachmentMapperTests.java

@@ -22,13 +22,13 @@ package org.elasticsearch.mapper.attachments;
 import org.elasticsearch.common.Base64;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.MapperTestUtils;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.DocumentMapperParser;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.core.DateFieldMapper;
 import org.elasticsearch.index.mapper.core.StringFieldMapper;
-import org.elasticsearch.mapper.attachments.AttachmentMapper;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.junit.After;
 import org.junit.Before;
@@ -48,7 +48,7 @@ public class MultifieldAttachmentMapperTests extends AttachmentUnitTestCase {
 
     @Before
     public void setupMapperParser() throws Exception {
-        mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY).documentMapperParser();
+        mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser();
 
     }
 
@@ -91,7 +91,7 @@ public class MultifieldAttachmentMapperTests extends AttachmentUnitTestCase {
         String bytes = Base64.encodeBytes(originalText.getBytes(StandardCharsets.ISO_8859_1));
         threadPool = new ThreadPool("testing-only");
 
-        MapperService mapperService = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY);
+        MapperService mapperService = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper());
 
         String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/multifield/multifield-mapping.json");
 

plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/SimpleAttachmentMapperTests.java

@@ -25,6 +25,7 @@ import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.MapperTestUtils;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.DocumentMapperParser;
 import org.elasticsearch.index.mapper.MapperService;
@@ -42,7 +43,7 @@ import static org.hamcrest.Matchers.*;
 public class SimpleAttachmentMapperTests extends AttachmentUnitTestCase {
 
     public void testSimpleMappings() throws Exception {
-        DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY).documentMapperParser();
+        DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser();
         String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/simple/test-mapping.json");
         DocumentMapper docMapper = mapperParser.parse(mapping);
         byte[] html = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/testXHTML.html");
@@ -69,9 +70,8 @@ public class SimpleAttachmentMapperTests extends AttachmentUnitTestCase {
 
     public void testContentBackcompat() throws Exception {
         DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(),
-            Settings.builder()
-                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id)
-            .build()).documentMapperParser();
+            Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build(),
+            getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser();
         String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/simple/test-mapping.json");
         DocumentMapper docMapper = mapperParser.parse(mapping);
         byte[] html = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/testXHTML.html");
@@ -86,7 +86,7 @@ public class SimpleAttachmentMapperTests extends AttachmentUnitTestCase {
      * test for https://github.com/elastic/elasticsearch-mapper-attachments/issues/179
      */
     public void testSimpleMappingsWithAllFields() throws Exception {
-        DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY).documentMapperParser();
+        DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser();
         String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/simple/test-mapping-all-fields.json");
         DocumentMapper docMapper = mapperParser.parse(mapping);
         byte[] html = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/testXHTML.html");
@@ -131,7 +131,7 @@ public class SimpleAttachmentMapperTests extends AttachmentUnitTestCase {
                 .endObject();
 
         byte[] mapping = mappingBuilder.bytes().toBytes();
-        MapperService mapperService = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY);
+        MapperService mapperService = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper());
         DocumentMapper docMapper = mapperService.parse("mail", new CompressedXContent(mapping), true);
         // this should not throw an exception
         mapperService.parse("mail", new CompressedXContent(docMapper.mapping().toString()), true);

plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/StandaloneRunner.java

@@ -30,10 +30,10 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.MapperTestUtils;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.DocumentMapperParser;
 import org.elasticsearch.index.mapper.ParseContext;
-import org.elasticsearch.mapper.attachments.AttachmentMapper;
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
@@ -46,6 +46,7 @@ import static org.elasticsearch.common.cli.CliToolConfig.Builder.cmd;
 import static org.elasticsearch.common.cli.CliToolConfig.Builder.option;
 import static org.elasticsearch.common.io.Streams.copy;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.mapper.attachments.AttachmentUnitTestCase.getIndicesModuleWithRegisteredAttachmentMapper;
 import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
 
 /**
@@ -88,7 +89,7 @@ public class StandaloneRunner extends CliTool {
             this.size = size;
             this.url = url;
             this.base64text = base64text;
-            DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(PathUtils.get("."), Settings.EMPTY).documentMapperParser(); // use CWD b/c it won't be used
+            DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(PathUtils.get("."), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser(); // use CWD b/c it won't be used
 
             String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/standalone/standalone-mapping.json");
             docMapper = mapperParser.parse(mapping);

plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/VariousDocTests.java

@@ -23,10 +23,10 @@ import org.apache.tika.io.IOUtils;
 import org.apache.tika.metadata.Metadata;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.MapperTestUtils;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.DocumentMapperParser;
 import org.elasticsearch.index.mapper.ParseContext;
-import org.elasticsearch.mapper.attachments.AttachmentMapper;
 import org.junit.Before;
 
 import java.io.IOException;
@@ -48,7 +48,7 @@ public class VariousDocTests extends AttachmentUnitTestCase {
 
     @Before
     public void createMapper() throws IOException {
-        DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY).documentMapperParser();
+        DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser();
 
         String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/various-doc/test-mapping.json");
         docMapper = mapperParser.parse(mapping);
@@ -93,7 +93,7 @@ public class VariousDocTests extends AttachmentUnitTestCase {
         assertParseable("text-in-english.txt");
         testMapper("text-in-english.txt", false);
     }
-    
+
     /**
      * Test for .epub
      */
@@ -129,7 +129,7 @@ public class VariousDocTests extends AttachmentUnitTestCase {
     protected void assertParseable(String filename) throws Exception {
         try (InputStream is = VariousDocTests.class.getResourceAsStream("/org/elasticsearch/index/mapper/attachment/test/sample-files/" + filename)) {
             byte bytes[] = IOUtils.toByteArray(is);
-            String parsedContent = TikaImpl.parse(bytes, new Metadata(), -1);  
+            String parsedContent = TikaImpl.parse(bytes, new Metadata(), -1);
             assertThat(parsedContent, not(isEmptyOrNullString()));
             logger.debug("extracted content: {}", parsedContent);
         }

plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MapperTestUtils.java → test-framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java

@@ -17,14 +17,12 @@
  * under the License.
  */
 
-package org.elasticsearch.mapper.attachments;
+package org.elasticsearch.index;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.AnalysisRegistry;
 import org.elasticsearch.index.analysis.AnalysisService;
 import org.elasticsearch.index.mapper.MapperService;
@@ -37,22 +35,29 @@ import java.io.IOException;
 import java.nio.file.Path;
 import java.util.Collections;
 
-class MapperTestUtils {
+
+public class MapperTestUtils {
 
     public static MapperService newMapperService(Path tempDir, Settings indexSettings) throws IOException {
-        Settings nodeSettings = Settings.builder()
-            .put("path.home", tempDir)
-            .build();
-        indexSettings = Settings.builder()
-            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
-            .put(indexSettings)
-            .build();
-        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(new Index("test"), indexSettings);
-        AnalysisService analysisService = new AnalysisRegistry(null, new Environment(nodeSettings)).build(idxSettings);
-        SimilarityService similarityService = new SimilarityService(idxSettings, Collections.emptyMap());
         IndicesModule indicesModule = new IndicesModule();
-        indicesModule.registerMapper(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser());
+        return newMapperService(tempDir, indexSettings, indicesModule);
+    }
+
+    public static MapperService newMapperService(Path tempDir, Settings settings, IndicesModule indicesModule) throws IOException {
+        Settings.Builder settingsBuilder = Settings.builder()
+            .put("path.home", tempDir)
+            .put(settings);
+        if (settings.get(IndexMetaData.SETTING_VERSION_CREATED) == null) {
+            settingsBuilder.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT);
+        }
+        Settings finalSettings = settingsBuilder.build();
         MapperRegistry mapperRegistry = indicesModule.getMapperRegistry();
-        return new MapperService(idxSettings, analysisService, similarityService, mapperRegistry);
+        IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(new Index("test"), finalSettings);
+        AnalysisService analysisService = new AnalysisRegistry(null, new Environment(finalSettings)).build(indexSettings);
+        SimilarityService similarityService = new SimilarityService(indexSettings, Collections.emptyMap());
+        return new MapperService(indexSettings,
+            analysisService,
+            similarityService,
+            mapperRegistry);
     }
 }