Mappings: Remove ability to disable _source field

Current features (eg. update API) and future features (eg. reindex API)
depend on _source. This change locks down the field so that
it can no longer be disabled. It also removes legacy settings
compress/compress_threshold.

closes #8142
closes #10915

docs/reference/mapping/fields/source-field.asciidoc

@@ -8,15 +8,6 @@ actual JSON that was used as the indexed document. It is not indexed
 <<search-search,search>>, the `_source` field is
 returned by default.
 
-Though very handy to have around, the source field does incur storage
-overhead within the index. For this reason, it can be disabled. For
-example:
-
-[source,js]
---------------------------------------------------
-{
-    "tweet" : {
-        "_source" : {"enabled" : false}
-    }
-}
---------------------------------------------------
+Many APIs may use the `_source` field. For example, the
+<<docs-update,Update API>>. To minimize the storage cost of
+`_source`, set `index.codec: best_compression` in index settings.

docs/reference/migration/migrate_2_0.asciidoc

@@ -272,6 +272,15 @@ to provide special features.  They now have limited configuration options.
 * `_field_names` configuration is limited to disabling the field.
 * `_size` configuration is limited to enabling the field.
 
+==== Source field limitations
+The `_source` field could previously be disabled dynamically. Since this field
+is a critical piece of many features like the Update API, it is no longer
+possible to disable.
+
+The options for `compress` and `compress_threshold` have also been removed.
+The source field is already compressed. To minimize the storage cost,
+set `index.codec: best_compression` in index settings.
+
 ==== Boolean fields
 
 Boolean fields used to have a string fielddata with `F` meaning `false` and `T`

src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java

@@ -145,15 +145,15 @@ public class SourceFieldMapper extends AbstractFieldMapper<byte[]> implements In
                 Map.Entry<String, Object> entry = iterator.next();
                 String fieldName = Strings.toUnderscoreCase(entry.getKey());
                 Object fieldNode = entry.getValue();
-                if (fieldName.equals("enabled")) {
+                if (fieldName.equals("enabled") && parserContext.indexVersionCreated().before(Version.V_2_0_0)) {
                     builder.enabled(nodeBooleanValue(fieldNode));
                     iterator.remove();
-                } else if (fieldName.equals("compress")) {
+                } else if (fieldName.equals("compress") && parserContext.indexVersionCreated().before(Version.V_2_0_0)) {
                     if (fieldNode != null) {
                         builder.compress(nodeBooleanValue(fieldNode));
                     }
                     iterator.remove();
-                } else if (fieldName.equals("compress_threshold")) {
+                } else if (fieldName.equals("compress_threshold") && parserContext.indexVersionCreated().before(Version.V_2_0_0)) {
                     if (fieldNode != null) {
                         if (fieldNode instanceof Number) {
                             builder.compressThreshold(((Number) fieldNode).longValue());

src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayTests.java

@@ -310,7 +310,7 @@ public class RecoveryFromGatewayTests extends ElasticsearchIntegrationTest {
 
                     logger.info("--> add some metadata, additional type and template");
                     client.admin().indices().preparePutMapping("test").setType("type2")
-                            .setSource(jsonBuilder().startObject().startObject("type2").startObject("_source").field("enabled", false).endObject().endObject().endObject())
+                            .setSource(jsonBuilder().startObject().startObject("type2").endObject().endObject())
                             .execute().actionGet();
                     client.admin().indices().preparePutTemplate("template_1")
                             .setTemplate("te*")

src/test/java/org/elasticsearch/get/GetActionTests.java

@@ -228,8 +228,9 @@ public class GetActionTests extends ElasticsearchIntegrationTest {
     }
 
     @Test
-    public void realtimeGetWithCompress() throws Exception {
-        assertAcked(prepareCreate("test").setSettings(ImmutableSettings.settingsBuilder().put("index.refresh_interval", -1))
+    public void realtimeGetWithCompressBackcompat() throws Exception {
+        assertAcked(prepareCreate("test")
+                .setSettings(ImmutableSettings.settingsBuilder().put("index.refresh_interval", -1).put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id))
                 .addMapping("type", jsonBuilder().startObject().startObject("type").startObject("_source").field("compress", true).endObject().endObject().endObject()));
         ensureGreen();
 
@@ -249,9 +250,8 @@ public class GetActionTests extends ElasticsearchIntegrationTest {
     @Test
     public void getFieldsWithDifferentTypes() throws Exception {
         assertAcked(prepareCreate("test").setSettings(ImmutableSettings.settingsBuilder().put("index.refresh_interval", -1))
-                .addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("_source").field("enabled", true).endObject().endObject().endObject())
+                .addMapping("type1", jsonBuilder().startObject().startObject("type1").endObject().endObject())
                 .addMapping("type2", jsonBuilder().startObject().startObject("type2")
-                        .startObject("_source").field("enabled", false).endObject()
                         .startObject("properties")
                         .startObject("str").field("type", "string").field("store", "yes").endObject()
                         .startObject("strs").field("type", "string").field("store", "yes").endObject()
@@ -339,7 +339,6 @@ public class GetActionTests extends ElasticsearchIntegrationTest {
                 .startObject("properties")
                 .startObject("field").field("type", "string").field("store", "yes").endObject()
                 .endObject()
-                .startObject("_source").field("enabled", false).endObject()
                 .endObject().endObject().string();
         assertAcked(prepareCreate("test")
                 .addMapping("type1", mapping1)
@@ -397,7 +396,7 @@ public class GetActionTests extends ElasticsearchIntegrationTest {
     }
 
     @Test
-    public void testThatGetFromTranslogShouldWorkWithExclude() throws Exception {
+    public void testThatGetFromTranslogShouldWorkWithExcludeBackcompat() throws Exception {
         String index = "test";
         String type = "type1";
 
@@ -431,7 +430,7 @@ public class GetActionTests extends ElasticsearchIntegrationTest {
     }
 
     @Test
-    public void testThatGetFromTranslogShouldWorkWithInclude() throws Exception {
+    public void testThatGetFromTranslogShouldWorkWithIncludeBackcompat() throws Exception {
         String index = "test";
         String type = "type1";
 
@@ -466,7 +465,7 @@ public class GetActionTests extends ElasticsearchIntegrationTest {
 
     @SuppressWarnings("unchecked")
     @Test
-    public void testThatGetFromTranslogShouldWorkWithIncludeExcludeAndFields() throws Exception {
+    public void testThatGetFromTranslogShouldWorkWithIncludeExcludeAndFieldsBackcompat() throws Exception {
         String index = "test";
         String type = "type1";
 
@@ -925,9 +924,6 @@ public class GetActionTests extends ElasticsearchIntegrationTest {
                 "  },\n" +
                 "  \"mappings\": {\n" +
                 "    \"doc\": {\n" +
-                "      \"_source\": {\n" +
-                "        \"enabled\": \"" + randomBoolean() + "\"\n" +
-                "      },\n" +
                 "      \"properties\": {\n" +
                 "        \"suggest\": {\n" +
                 "          \"type\": \"completion\"\n" +
@@ -970,9 +966,6 @@ public class GetActionTests extends ElasticsearchIntegrationTest {
                 "  \"mappings\": {\n" +
                 "    \"parentdoc\": {},\n" +
                 "    \"doc\": {\n" +
-                "      \"_source\": {\n" +
-                "        \"enabled\": " + randomBoolean() + "\n" +
-                "      },\n" +
                 "      \"_parent\": {\n" +
                 "        \"type\": \"parentdoc\"\n" +
                 "      },\n" +
@@ -1002,7 +995,7 @@ public class GetActionTests extends ElasticsearchIntegrationTest {
     }
 
     @Test
-    public void testUngeneratedFieldsPartOfSourceUnstoredSourceDisabled() throws IOException {
+    public void testUngeneratedFieldsPartOfSourceUnstoredSourceDisabledBackcompat() throws IOException {
         indexSingleDocumentWithUngeneratedFieldsThatArePartOf_source(false, false);
         String[] fieldsList = {};
         // before refresh - document is only in translog
@@ -1016,7 +1009,7 @@ public class GetActionTests extends ElasticsearchIntegrationTest {
     }
 
     @Test
-    public void testUngeneratedFieldsPartOfSourceEitherStoredOrSourceEnabled() throws IOException {
+    public void testUngeneratedFieldsPartOfSourceEitherStoredOrSourceEnabledBackcompat() throws IOException {
         boolean stored = randomBoolean();
         boolean sourceEnabled = true;
         if (stored) {
@@ -1039,7 +1032,8 @@ public class GetActionTests extends ElasticsearchIntegrationTest {
         String createIndexSource = "{\n" +
                 "  \"settings\": {\n" +
                 "    \"index.translog.disable_flush\": true,\n" +
-                "    \"refresh_interval\": \"-1\"\n" +
+                "    \"refresh_interval\": \"-1\",\n" +
+                "    \"" + IndexMetaData.SETTING_VERSION_CREATED + "\": " + Version.V_1_4_2.id + "\n" +
                 "  },\n" +
                 "  \"mappings\": {\n" +
                 "    \"doc\": {\n" +
@@ -1161,7 +1155,8 @@ public class GetActionTests extends ElasticsearchIntegrationTest {
         String createIndexSource = "{\n" +
                 "  \"settings\": {\n" +
                 "    \"index.translog.disable_flush\": true,\n" +
-                "    \"refresh_interval\": \"-1\"\n" +
+                "    \"refresh_interval\": \"-1\",\n" +
+                "    \"" + IndexMetaData.SETTING_VERSION_CREATED + "\": " + Version.V_1_4_2.id + "\n" +
                 "  },\n" +
                 "  \"mappings\": {\n" +
                 "    \"doc\": {\n" +
@@ -1215,7 +1210,8 @@ public class GetActionTests extends ElasticsearchIntegrationTest {
         String createIndexSource = "{\n" +
                 "  \"settings\": {\n" +
                 "    \"index.translog.disable_flush\": true,\n" +
-                "    \"refresh_interval\": \"-1\"\n" +
+                "    \"refresh_interval\": \"-1\",\n" +
+                "    \"" + IndexMetaData.SETTING_VERSION_CREATED + "\": " + Version.V_1_4_2.id + "\n" +
                 "  },\n" +
                 "  \"mappings\": {\n" +
                 "    \"doc\": {\n" +

src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java

@@ -423,7 +423,6 @@ public class SimpleAllMapperTests extends ElasticsearchSingleNodeTest {
         //just pick some example from DocumentMapperParser.rootTypeParsers
         rootTypes.put(SizeFieldMapper.NAME, "{\"enabled\" : true}");
         rootTypes.put(IndexFieldMapper.NAME, "{\"enabled\" : true}");
-        rootTypes.put(SourceFieldMapper.NAME, "{\"enabled\" : true}");
         rootTypes.put("include_in_all", "true");
         rootTypes.put("dynamic_date_formats", "[\"yyyy-MM-dd\", \"dd-MM-yyyy\"]");
         rootTypes.put("numeric_detection", "true");

src/test/java/org/elasticsearch/index/mapper/source/CompressSourceMappingTests.java

@@ -20,7 +20,11 @@
 package org.elasticsearch.index.mapper.source;
 
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.compress.CompressorFactory;
+import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.ParsedDocument;
@@ -33,6 +37,7 @@ import static org.hamcrest.Matchers.equalTo;
  *
  */
 public class CompressSourceMappingTests extends ElasticsearchSingleNodeTest {
+    Settings settings = ImmutableSettings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build();
 
     @Test
     public void testCompressDisabled() throws Exception {
@@ -40,7 +45,7 @@ public class CompressSourceMappingTests extends ElasticsearchSingleNodeTest {
                 .startObject("_source").field("compress", false).endObject()
                 .endObject().endObject().string();
 
-        DocumentMapper documentMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping);
+        DocumentMapper documentMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping);
 
         ParsedDocument doc = documentMapper.parse("type", "1", XContentFactory.jsonBuilder().startObject()
                 .field("field1", "value1")
@@ -56,7 +61,7 @@ public class CompressSourceMappingTests extends ElasticsearchSingleNodeTest {
                 .startObject("_source").field("compress", true).endObject()
                 .endObject().endObject().string();
 
-        DocumentMapper documentMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping);
+        DocumentMapper documentMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping);
 
         ParsedDocument doc = documentMapper.parse("type", "1", XContentFactory.jsonBuilder().startObject()
                 .field("field1", "value1")
@@ -73,7 +78,7 @@ public class CompressSourceMappingTests extends ElasticsearchSingleNodeTest {
                 .startObject("_source").field("compress_threshold", "200b").endObject()
                 .endObject().endObject().string();
 
-        DocumentMapper documentMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping);
+        DocumentMapper documentMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping);
 
         ParsedDocument doc = documentMapper.parse("type", "1", XContentFactory.jsonBuilder().startObject()
                 .field("field1", "value1")

src/test/java/org/elasticsearch/index/mapper/source/DefaultSourceMappingTests.java

@@ -42,6 +42,7 @@ import java.util.Map;
 import static org.hamcrest.Matchers.*;
 
 public class DefaultSourceMappingTests extends ElasticsearchSingleNodeTest {
+    Settings backcompatSettings = ImmutableSettings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build();
 
     public void testNoFormat() throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
@@ -85,12 +86,12 @@ public class DefaultSourceMappingTests extends ElasticsearchSingleNodeTest {
         assertThat(XContentFactory.xContentType(doc.source()), equalTo(XContentType.JSON));
     }
 
-    public void testJsonFormatCompressed() throws Exception {
+    public void testJsonFormatCompressedBackcompat() throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("_source").field("format", "json").field("compress", true).endObject()
                 .endObject().endObject().string();
 
-        DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
+        DocumentMapperParser parser = createIndex("test", backcompatSettings).mapperService().documentMapperParser();
         DocumentMapper documentMapper = parser.parse(mapping);
         ParsedDocument doc = documentMapper.parse("type", "1", XContentFactory.jsonBuilder().startObject()
                 .field("field", "value")
@@ -122,8 +123,7 @@ public class DefaultSourceMappingTests extends ElasticsearchSingleNodeTest {
             assertTrue(e.getMessage().contains("unsupported parameters"));
         }
 
-        Settings settings = ImmutableSettings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build();
-        DocumentMapper documentMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping);
+        DocumentMapper documentMapper = createIndex("test", backcompatSettings).mapperService().documentMapperParser().parse(mapping);
 
         ParsedDocument doc = documentMapper.parse("type", "1", XContentFactory.jsonBuilder().startObject()
             .startObject("path1").field("field1", "value1").endObject()
@@ -148,8 +148,7 @@ public class DefaultSourceMappingTests extends ElasticsearchSingleNodeTest {
             assertTrue(e.getMessage().contains("unsupported parameters"));
         }
 
-        Settings settings = ImmutableSettings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build();
-        DocumentMapper documentMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping);
+        DocumentMapper documentMapper = createIndex("test", backcompatSettings).mapperService().documentMapperParser().parse(mapping);
 
         ParsedDocument doc = documentMapper.parse("type", "1", XContentFactory.jsonBuilder().startObject()
             .startObject("path1").field("field1", "value1").endObject()
@@ -162,12 +161,12 @@ public class DefaultSourceMappingTests extends ElasticsearchSingleNodeTest {
         assertThat(sourceAsMap.containsKey("path2"), equalTo(true));
     }
 
-    public void testDefaultMappingAndNoMapping() throws Exception {
+    public void testDefaultMappingAndNoMappingBackcompat() throws Exception {
         String defaultMapping = XContentFactory.jsonBuilder().startObject().startObject(MapperService.DEFAULT_MAPPING)
                 .startObject("_source").field("enabled", false).endObject()
                 .endObject().endObject().string();
 
-        DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
+        DocumentMapperParser parser = createIndex("test", backcompatSettings).mapperService().documentMapperParser();
         DocumentMapper mapper = parser.parse("my_type", null, defaultMapping);
         assertThat(mapper.type(), equalTo("my_type"));
         assertThat(mapper.sourceMapper().enabled(), equalTo(false));
@@ -190,7 +189,7 @@ public class DefaultSourceMappingTests extends ElasticsearchSingleNodeTest {
         }
     }
 
-    public void testDefaultMappingAndWithMappingOverride() throws Exception {
+    public void testDefaultMappingAndWithMappingOverrideBackcompat() throws Exception {
         String defaultMapping = XContentFactory.jsonBuilder().startObject().startObject(MapperService.DEFAULT_MAPPING)
                 .startObject("_source").field("enabled", false).endObject()
                 .endObject().endObject().string();
@@ -199,17 +198,17 @@ public class DefaultSourceMappingTests extends ElasticsearchSingleNodeTest {
                 .startObject("_source").field("enabled", true).endObject()
                 .endObject().endObject().string();
 
-        DocumentMapper mapper = createIndex("test").mapperService().documentMapperParser().parse("my_type", mapping, defaultMapping);
+        DocumentMapper mapper = createIndex("test", backcompatSettings).mapperService().documentMapperParser().parse("my_type", mapping, defaultMapping);
         assertThat(mapper.type(), equalTo("my_type"));
         assertThat(mapper.sourceMapper().enabled(), equalTo(true));
     }
 
-    public void testDefaultMappingAndNoMappingWithMapperService() throws Exception {
+    public void testDefaultMappingAndNoMappingWithMapperServiceBackcompat() throws Exception {
         String defaultMapping = XContentFactory.jsonBuilder().startObject().startObject(MapperService.DEFAULT_MAPPING)
                 .startObject("_source").field("enabled", false).endObject()
                 .endObject().endObject().string();
 
-        MapperService mapperService = createIndex("test").mapperService();
+        MapperService mapperService = createIndex("test", backcompatSettings).mapperService();
         mapperService.merge(MapperService.DEFAULT_MAPPING, new CompressedString(defaultMapping), true);
 
         DocumentMapper mapper = mapperService.documentMapperWithAutoCreate("my_type").v1();
@@ -217,12 +216,12 @@ public class DefaultSourceMappingTests extends ElasticsearchSingleNodeTest {
         assertThat(mapper.sourceMapper().enabled(), equalTo(false));
     }
 
-    public void testDefaultMappingAndWithMappingOverrideWithMapperService() throws Exception {
+    public void testDefaultMappingAndWithMappingOverrideWithMapperServiceBackcompat() throws Exception {
         String defaultMapping = XContentFactory.jsonBuilder().startObject().startObject(MapperService.DEFAULT_MAPPING)
                 .startObject("_source").field("enabled", false).endObject()
                 .endObject().endObject().string();
 
-        MapperService mapperService = createIndex("test").mapperService();
+        MapperService mapperService = createIndex("test", backcompatSettings).mapperService();
         mapperService.merge(MapperService.DEFAULT_MAPPING, new CompressedString(defaultMapping), true);
 
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("my_type")

src/test/java/org/elasticsearch/index/mapper/typelevels/ParseMappingTypeLevelTests.java

@@ -27,24 +27,22 @@ import org.junit.Test;
 
 import static org.hamcrest.Matchers.equalTo;
 
-/**
- *
- */
+// TODO: move this test...it doesn't need to be by itself
 public class ParseMappingTypeLevelTests extends ElasticsearchSingleNodeTest {
 
     @Test
     public void testTypeLevel() throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("_source").field("enabled", false).endObject()
+                .startObject("_index").field("enabled", true).endObject()
                 .endObject().endObject().string();
 
         DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
         DocumentMapper mapper = parser.parse("type", mapping);
         assertThat(mapper.type(), equalTo("type"));
-        assertThat(mapper.sourceMapper().enabled(), equalTo(false));
+        assertThat(mapper.indexMapper().enabled(), equalTo(true));
 
         mapper = parser.parse(mapping);
         assertThat(mapper.type(), equalTo("type"));
-        assertThat(mapper.sourceMapper().enabled(), equalTo(false));
+        assertThat(mapper.indexMapper().enabled(), equalTo(true));
     }
 }

src/test/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationTests.java

@@ -126,14 +126,14 @@ public class UpdateMappingIntegrationTests extends ElasticsearchIntegrationTest
         client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().execute().actionGet();
 
         PutMappingResponse putMappingResponse = client().admin().indices().preparePutMapping("test").setType("doc")
-                .setSource("{\"_source\":{\"enabled\":false},\"properties\":{\"date\":{\"type\":\"integer\"}}}")
+                .setSource("{\"properties\":{\"date\":{\"type\":\"integer\"}}}")
                 .execute().actionGet();
 
         assertThat(putMappingResponse.isAcknowledged(), equalTo(true));
 
         GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings("test").execute().actionGet();
         assertThat(getMappingsResponse.mappings().get("test").get("doc").source().toString(),
-                equalTo("{\"doc\":{\"_source\":{\"enabled\":false},\"properties\":{\"date\":{\"type\":\"integer\"}}}}"));
+                equalTo("{\"doc\":{\"properties\":{\"date\":{\"type\":\"integer\"}}}}"));
     }
 
     @Test(expected = MergeMappingException.class)

src/test/java/org/elasticsearch/search/compress/SearchSourceCompressTests.java

@@ -19,25 +19,26 @@
 
 package org.elasticsearch.search.compress;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.compress.CompressorFactory;
 import org.elasticsearch.common.compress.lzf.LZFCompressor;
+import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.test.ElasticsearchIntegrationTest;
+import org.elasticsearch.test.ElasticsearchSingleNodeTest;
 import org.junit.Test;
 
 import java.io.IOException;
 
 import static org.hamcrest.Matchers.equalTo;
 
-/**
- *
- */
-public class SearchSourceCompressTests  extends ElasticsearchIntegrationTest {
+public class SearchSourceCompressTests  extends ElasticsearchSingleNodeTest {
     
     @Test
     public void testSourceCompressionLZF() throws IOException {
@@ -53,7 +54,8 @@ public class SearchSourceCompressTests  extends ElasticsearchIntegrationTest {
         } catch (Exception e) {
             // ignore
         }
-        createIndex("test");
+        Settings settings = ImmutableSettings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build();
+        createIndex("test", settings);
         client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().execute().actionGet();
 
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")

src/test/java/org/elasticsearch/search/highlight/HighlighterSearchTests.java

@@ -23,10 +23,11 @@ import com.google.common.base.Joiner;
 import com.google.common.collect.Iterables;
 
 import org.apache.lucene.util.LuceneTestCase.Slow;
-import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.ImmutableSettings.Builder;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -90,9 +91,6 @@ import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.startsWith;
 
-/**
- *
- */
 @Slow
 public class HighlighterSearchTests extends ElasticsearchIntegrationTest {
 
@@ -564,9 +562,10 @@ public class HighlighterSearchTests extends ElasticsearchIntegrationTest {
     }
 
     @Test
-    public void testForceSourceWithSourceDisabled() throws Exception {
+    public void testForceSourceWithSourceDisabledBackcompat() throws Exception {
 
         assertAcked(prepareCreate("test")
+                .setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id)
                 .addMapping("type1", jsonBuilder().startObject().startObject("type1")
                         .startObject("_source").field("enabled", false).endObject()
                         .startObject("properties")
@@ -2180,8 +2179,6 @@ public class HighlighterSearchTests extends ElasticsearchIntegrationTest {
     public void testPostingsHighlighterMultiMapperWithStore() throws Exception {
         assertAcked(prepareCreate("test")
                 .addMapping("type1", jsonBuilder().startObject().startObject("type1")
-                        //just to make sure that we hit the stored fields rather than the _source
-                        .startObject("_source").field("enabled", false).endObject()
                         .startObject("properties")
                         .startObject("title").field("type", "multi_field").startObject("fields")
                         .startObject("title").field("type", "string").field("store", "yes").field("index_options", "offsets").field("analyzer", "classic").endObject()
@@ -2199,7 +2196,6 @@ public class HighlighterSearchTests extends ElasticsearchIntegrationTest {
 
         assertHitCount(searchResponse, 1l);
         SearchHit hit = searchResponse.getHits().getAt(0);
-        assertThat(hit.source(), nullValue());
         //stopwords are not highlighted since not indexed
         assertHighlight(hit, "title", 0, 1, equalTo("this is a <em>test</em> ."));
 

src/test/java/org/elasticsearch/search/innerhits/InnerHitsTests.java

@@ -697,8 +697,9 @@ public class InnerHitsTests extends ElasticsearchIntegrationTest {
     }
 
     @Test
-    public void testNestedInnerHitsWithStoredFieldsAndNoSource() throws Exception {
+    public void testNestedInnerHitsWithStoredFieldsAndNoSourceBackcompat() throws Exception {
         assertAcked(prepareCreate("articles")
+                .setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id)
                 .addMapping("article", jsonBuilder().startObject()
                                 .startObject("_source").field("enabled", false).endObject()
                                 .startObject("properties")
@@ -735,8 +736,9 @@ public class InnerHitsTests extends ElasticsearchIntegrationTest {
     }
 
     @Test
-    public void testNestedInnerHitsWithHighlightOnStoredField() throws Exception {
+    public void testNestedInnerHitsWithHighlightOnStoredFieldBackcompat() throws Exception {
         assertAcked(prepareCreate("articles")
+                .setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id)
                         .addMapping("article", jsonBuilder().startObject()
                                         .startObject("_source").field("enabled", false).endObject()
                                             .startObject("properties")
@@ -773,7 +775,7 @@ public class InnerHitsTests extends ElasticsearchIntegrationTest {
     }
 
     @Test
-    public void testNestedInnerHitsWithExcludeSource() throws Exception {
+    public void testNestedInnerHitsWithExcludeSourceBackcompat() throws Exception {
         assertAcked(prepareCreate("articles").setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id)
                         .addMapping("article", jsonBuilder().startObject()
                                         .startObject("_source").field("excludes", new String[]{"comments"}).endObject()
@@ -811,7 +813,7 @@ public class InnerHitsTests extends ElasticsearchIntegrationTest {
     }
 
     @Test
-    public void testNestedInnerHitsHiglightWithExcludeSource() throws Exception {
+    public void testNestedInnerHitsHiglightWithExcludeSourceBackcompat() throws Exception {
         assertAcked(prepareCreate("articles").setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id)
                         .addMapping("article", jsonBuilder().startObject()
                                         .startObject("_source").field("excludes", new String[]{"comments"}).endObject()

src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java

@@ -393,11 +393,6 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
                             .field("enabled", randomBoolean())
                             .endObject();
                 }
-                if (randomBoolean()) {
-                    mappings.startObject(SourceFieldMapper.NAME)
-                            .field("compress", randomBoolean())
-                            .endObject();
-                }
                 mappings.startArray("dynamic_templates")
                         .startObject()
                     .startObject("template-strings")