Merge remote-tracking branch 'dakrone/exclude-numerics-from-all'

core/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java

@@ -496,7 +496,10 @@ public class DateFieldMapper extends FieldMapper {
     }
 
     @Override
-    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
+    protected void parseCreateField(ParseContext originalContext, List<Field> fields) throws IOException {
+        // Date fields, by default, will not be included in _all
+        final ParseContext context = originalContext.setIncludeInAllDefault(false);
+
         String dateAsString;
         if (context.externalValueSet()) {
             Object dateAsObject = context.externalValue();

core/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java

@@ -110,7 +110,10 @@ public class GeoPointFieldMapper extends BaseGeoPointFieldMapper  {
     }
 
     @Override
-    protected void parse(ParseContext context, GeoPoint point, String geoHash) throws IOException {
+    protected void parse(ParseContext originalContext, GeoPoint point, String geoHash) throws IOException {
+        // Geopoint fields, by default, will not be included in _all
+        final ParseContext context = originalContext.setIncludeInAllDefault(false);
+
         if (ignoreMalformed.value() == false) {
             if (point.lat() > 90.0 || point.lat() < -90.0) {
                 throw new IllegalArgumentException("illegal latitude value [" + point.lat() + "] for " + name());

core/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java

@@ -430,7 +430,10 @@ public class GeoShapeFieldMapper extends FieldMapper {
     }
 
     @Override
-    public Mapper parse(ParseContext context) throws IOException {
+    public Mapper parse(ParseContext originalContext) throws IOException {
+        // Numeric fields, by default, will not be included in _all
+        final ParseContext context = originalContext.setIncludeInAllDefault(false);
+
         try {
             Shape shape = context.parseExternalValue(Shape.class);
             if (shape == null) {

core/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java

@@ -285,7 +285,10 @@ public class IpFieldMapper extends FieldMapper {
     }
 
     @Override
-    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
+    protected void parseCreateField(ParseContext originalContext, List<Field> fields) throws IOException {
+        // IP fields, by default, will not be included in _all
+        final ParseContext context = originalContext.setIncludeInAllDefault(false);
+
         Object addressAsObject;
         if (context.externalValueSet()) {
             addressAsObject = context.externalValue();

core/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java

@@ -895,7 +895,10 @@ public class NumberFieldMapper extends FieldMapper {
     }
 
     @Override
-    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
+    protected void parseCreateField(ParseContext originalContext, List<Field> fields) throws IOException {
+        // Numeric fields, by default, will not be included in _all
+        final ParseContext context = originalContext.setIncludeInAllDefault(false);
+
         XContentParser parser = context.parser();
         Object value;
         Number numericValue = null;

core/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java

@@ -364,7 +364,10 @@ public class ScaledFloatFieldMapper extends FieldMapper {
     }
 
     @Override
-    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
+    protected void parseCreateField(ParseContext originalContext, List<Field> fields) throws IOException {
+        // Numeric fields, by default, will not be included in _all
+        final ParseContext context = originalContext.setIncludeInAllDefault(false);
+
         XContentParser parser = context.parser();
         Object value;
         Number numericValue = null;

core/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java

@@ -178,7 +178,8 @@ public class DateFieldMapperTests extends ESSingleNodeTestCase {
 
     public void testIncludeInAll() throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("field").field("type", "date").endObject().endObject()
+                .startObject("properties").startObject("field").field("type", "date")
+                .field("include_in_all", true).endObject().endObject()
                 .endObject().endObject().string();
 
         DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
@@ -196,8 +197,7 @@ public class DateFieldMapperTests extends ESSingleNodeTestCase {
         assertEquals("2016-03-11", fields[0].stringValue());
 
         mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("field").field("type", "date")
-                .field("include_in_all", false).endObject().endObject()
+                .startObject("properties").startObject("field").field("type", "date").endObject().endObject()
                 .endObject().endObject().string();
 
         mapper = parser.parse("type", new CompressedXContent(mapping));

core/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java

@@ -186,7 +186,8 @@ public class IpFieldMapperTests extends ESSingleNodeTestCase {
 
     public void testIncludeInAll() throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("field").field("type", "ip").endObject().endObject()
+                .startObject("properties").startObject("field").field("type", "ip")
+                .field("include_in_all", true).endObject().endObject()
                 .endObject().endObject().string();
 
         DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
@@ -204,8 +205,7 @@ public class IpFieldMapperTests extends ESSingleNodeTestCase {
         assertEquals("::1", fields[0].stringValue());
 
         mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("field").field("type", "ip")
-                .field("include_in_all", false).endObject().endObject()
+                .startObject("properties").startObject("field").field("type", "ip").endObject().endObject()
                 .endObject().endObject().string();
 
         mapper = parser.parse("type", new CompressedXContent(mapping));

core/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java

@@ -264,7 +264,8 @@ public class NumberFieldMapperTests extends ESSingleNodeTestCase {
 
     public void doTestIncludeInAll(String type) throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("field").field("type", type).endObject().endObject()
+                .startObject("properties").startObject("field").field("type", type)
+                .field("include_in_all", true).endObject().endObject()
                 .endObject().endObject().string();
 
         DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
@@ -282,8 +283,7 @@ public class NumberFieldMapperTests extends ESSingleNodeTestCase {
         assertEquals("123", fields[0].stringValue());
 
         mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("field").field("type", type)
-                .field("include_in_all", false).endObject().endObject()
+                .startObject("properties").startObject("field").field("type", type).endObject().endObject()
                 .endObject().endObject().string();
 
         mapper = parser.parse("type", new CompressedXContent(mapping));

core/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java

@@ -249,7 +249,8 @@ public class ScaledFloatFieldMapperTests extends ESSingleNodeTestCase {
     public void testIncludeInAll() throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("properties").startObject("field").field("type", "scaled_float")
-                .field("scaling_factor", 10.0).endObject().endObject()
+                .field("scaling_factor", 10.0)
+                .field("include_in_all", true).endObject().endObject()
                 .endObject().endObject().string();
 
         DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
@@ -268,8 +269,7 @@ public class ScaledFloatFieldMapperTests extends ESSingleNodeTestCase {
 
         mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("properties").startObject("field")
-                .field("type", "scaled_float").field("scaling_factor", 10.0)
-                .field("include_in_all", false).endObject().endObject()
+                .field("type", "scaled_float").field("scaling_factor", 10.0).endObject().endObject()
                 .endObject().endObject().string();
 
         mapper = parser.parse("type", new CompressedXContent(mapping));

docs/plugins/mapper-attachments.asciidoc

@@ -90,10 +90,10 @@ If you get a hit for your indexed document, the plugin should be installed and w
   "took": 53,
   "hits": {
     "total": 1,
-    "max_score": 0.25811607,
+    "max_score": 0.28582606,
     "hits": [
       {
-        "_score": 0.25811607,
+        "_score": 0.28582606,
         "_index": "trying-out-mapper-attachments",
         "_type": "person",
         "_id": "1",

docs/reference/mapping/fields/all-field.asciidoc

@@ -1,14 +1,14 @@
 [[mapping-all-field]]
 === `_all` field
 
-The `_all` field is a special _catch-all_ field which concatenates the values
-of all of the other fields into one big string, using space as a delimiter, which is then
-<<analysis,analyzed>> and indexed, but not stored.  This means that it can be
-searched, but not retrieved.
+The `_all` field is a special _catch-all_ field which concatenates the values of
+all of the other string fields into one big string, using space as a delimiter,
+which is then <<analysis,analyzed>> and indexed, but not stored. This means that
+it can be searched, but not retrieved.
 
-The `_all` field allows you to search for values in documents without knowing
-which field contains the value.  This makes it a useful option when getting
-started with a new dataset. For instance:
+The `_all` field allows you to search for string values in documents without
+knowing which field contains the value. This makes it a useful option when
+getting started with a new dataset. For instance:
 
 [source,js]
 --------------------------------
@@ -16,29 +16,28 @@ PUT my_index/user/1 <1>
 {
   "first_name":    "John",
   "last_name":     "Smith",
-  "date_of_birth": "1970-10-24"
+  "place_of_birth": "New York City"
 }
 
 GET my_index/_search
 {
   "query": {
     "match": {
-      "_all": "john smith 1970"
+      "_all": "john smith new york"
     }
   }
 }
 --------------------------------
 // CONSOLE
-<1> The `_all` field will contain the terms: [ `"john"`, `"smith"`, `"1970"`, `"10"`, `"24"` ]
+<1> The `_all` field will contain the terms: [ `"john"`, `"smith"`, `"new"`, `"york"`, `"city"` ]
 
 [NOTE]
-.All values treated as strings
+.Only string values are added to _all
 =============================================================================
 
-The `date_of_birth` field in the above example is recognised as a `date` field
-and so will index a single term representing `1970-10-24 00:00:00 UTC`. The
-`_all` field, however, treats all values as strings, so the date value is
-indexed as the three string terms: `"1970"`, `"24"`, `"10"`.
+If a `date_of_birth` field mapped as a date were used, or an `age` field that
+was an integer were added, they would not be included in the `_all` field, as
+`_all` only contains content from _string_ fields.
 
 It is important to note that the `_all` field combines the original values
 from each field as a string. It does not combine the _terms_ from each field.
@@ -72,7 +71,7 @@ GET _search
 {
   "query": {
     "query_string": {
-      "query": "john smith 1970"
+      "query": "john smith new york"
     }
   }
 }
@@ -84,7 +83,7 @@ requests>> (which is rewritten to a `query_string` query internally):
 
 [source,js]
 --------------------------------
-GET _search?q=john+smith+1970
+GET _search?q=john+smith+new+york
 --------------------------------
 
 Other queries, such as the <<query-dsl-match-query,`match`>> and

docs/reference/migration/migrate_5_0/mapping.asciidoc

@@ -158,6 +158,18 @@ of the 4 bytes used previously. While this will make the index much more
 space-efficient, it also means that index time boosts will be less accurately
 encoded.
 
+==== `include_in_all` default changed to false for numeric/date/ip/geo types
+
+The default unset value for numeric types for the `include_in_all` mapping
+option has been changed from true to false.
+
+This includes:
+
+- All regular numeric types such as int, long, float, scaled-float, double
+- IP addresses
+- Dates
+- Geopoints and Geoshapes
+
 ==== `_ttl` and `_timestamp` cannot be created
 
 You can no longer create indexes with `_ttl` or `_timestamp` enabled. Indexes