Make the `index` property a boolean.

With the split of `string` into `text` and `keyword`, the `index` property can
only have two values and should be a boolean.

core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java

@@ -398,7 +398,7 @@ public abstract class FieldMapper extends Mapper implements Cloneable {
         boolean defaultIndexed = defaultFieldType.indexOptions() != IndexOptions.NONE;
         if (includeDefaults || indexed != defaultIndexed ||
             fieldType().tokenized() != defaultFieldType.tokenized()) {
-            builder.field("index", indexTokenizeOptionToString(indexed, fieldType().tokenized()));
+            builder.field("index", indexTokenizeOption(indexed, fieldType().tokenized()));
         }
         if (includeDefaults || fieldType().stored() != defaultFieldType.stored()) {
             builder.field("store", fieldType().stored());
@@ -495,14 +495,9 @@ public abstract class FieldMapper extends Mapper implements Cloneable {
         }
     }
 
-    protected static String indexTokenizeOptionToString(boolean indexed, boolean tokenized) {
-        if (!indexed) {
-            return "no";
-        } else if (tokenized) {
-            return "analyzed";
-        } else {
-            return "not_analyzed";
-        }
+    /* Only protected so that string can override it */
+    protected Object indexTokenizeOption(boolean indexed, boolean tokenized) {
+        return indexed;
     }
 
     protected boolean hasCustomFieldDataSettings() {

core/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java

@@ -147,6 +147,27 @@ public class StringFieldMapper extends FieldMapper implements AllFieldMapper.Inc
         @Override
         public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
             StringFieldMapper.Builder builder = stringField(name);
+            // hack for the fact that string can't just accept true/false for
+            // the index property and still accepts no/not_analyzed/analyzed
+            final Object index = node.remove("index");
+            if (index != null) {
+                final String normalizedIndex = Strings.toUnderscoreCase(index.toString());
+                switch (normalizedIndex) {
+                case "analyzed":
+                    builder.tokenized(true);
+                    node.put("index", true);
+                    break;
+                case "not_analyzed":
+                    builder.tokenized(false);
+                    node.put("index", true);
+                    break;
+                case "no":
+                    node.put("index", false);
+                    break;
+                default:
+                    throw new IllegalArgumentException("Can't parse [index] value [" + index + "], expected [true], [false], [no], [not_analyzed] or [analyzed]");
+                }
+            }
             parseTextField(builder, name, node, parserContext);
             for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
                 Map.Entry<String, Object> entry = iterator.next();
@@ -369,6 +390,17 @@ public class StringFieldMapper extends FieldMapper implements AllFieldMapper.Inc
         this.ignoreAbove = ((StringFieldMapper) mergeWith).ignoreAbove;
     }
 
+    @Override
+    protected String indexTokenizeOption(boolean indexed, boolean tokenized) {
+        if (!indexed) {
+            return "no";
+        } else if (tokenized) {
+            return "analyzed";
+        } else {
+            return "not_analyzed";
+        }
+    }
+
     @Override
     protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
         super.doXContentBody(builder, includeDefaults, params);

core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java

@@ -172,7 +172,7 @@ public class TypeParsers {
                 builder.store(parseStore(name, propNode.toString(), parserContext));
                 iterator.remove();
             } else if (propName.equals("index")) {
-                parseIndex(name, propNode.toString(), builder);
+                builder.index(parseIndex(name, propNode.toString(), parserContext));
                 iterator.remove();
             } else if (propName.equals(DOC_VALUES)) {
                 builder.docValues(nodeBooleanValue(propNode, parserContext));
@@ -328,18 +328,29 @@ public class TypeParsers {
         }
     }
 
-    public static void parseIndex(String fieldName, String index, FieldMapper.Builder builder) throws MapperParsingException {
-        index = Strings.toUnderscoreCase(index);
-        if ("no".equals(index)) {
-            builder.index(false);
-        } else if ("not_analyzed".equals(index)) {
-            builder.index(true);
-            builder.tokenized(false);
-        } else if ("analyzed".equals(index)) {
-            builder.index(true);
-            builder.tokenized(true);
+    public static boolean parseIndex(String fieldName, String index, Mapper.TypeParser.ParserContext parserContext) throws MapperParsingException {
+        if (parserContext.indexVersionCreated().onOrAfter(Version.V_3_0_0)) {
+            switch (index) {
+            case "true":
+                return true;
+            case "false":
+                return false;
+            default:
+                throw new IllegalArgumentException("Can't parse [index] value [" + index + "], expected [true] or [false]");
+            }
         } else {
-            throw new MapperParsingException("wrong value for index [" + index + "] for field [" + fieldName + "]");
+            final String normalizedIndex = Strings.toUnderscoreCase(index);
+            switch (normalizedIndex) {
+            case "true":
+            case "not_analyzed":
+            case "analyzed":
+                return true;
+            case "false":
+            case "no":
+                return false;
+            default:
+                throw new IllegalArgumentException("Can't parse [index] value [" + index + "], expected [true], [false], [no], [not_analyzed] or [analyzed]");
+            }
         }
     }
 

core/src/test/java/org/elasticsearch/index/mapper/numeric/SimpleNumericTests.java

@@ -23,6 +23,7 @@ import org.apache.lucene.analysis.NumericTokenStream;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
@@ -354,6 +355,60 @@ public class SimpleNumericTests extends ESSingleNodeTestCase {
         assertEquals(DocValuesType.NONE, SimpleStringMappingTests.docValuesType(doc, "double2"));
     }
 
+    public void testUnIndex() throws IOException {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties")
+                .startObject("int")
+                    .field("type", "integer")
+                    .field("index", false)
+                .endObject()
+                .startObject("double")
+                    .field("type", "double")
+                    .field("index", false)
+                .endObject()
+                .endObject()
+                .endObject().endObject().string();
+
+        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+
+        assertEquals("{\"type\":{\"properties\":{\"double\":{\"type\":\"double\",\"index\":false},\"int\":{\"type\":\"integer\",\"index\":false}}}}",
+                defaultMapper.mapping().toString());
+
+        ParsedDocument parsedDoc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
+                .startObject()
+                .field("int", "1234")
+                .field("double", "1234")
+                .endObject()
+                .bytes());
+        final Document doc = parsedDoc.rootDoc();
+        for (IndexableField field : doc.getFields("int")) {
+            assertEquals(IndexOptions.NONE, field.fieldType().indexOptions());
+        }
+        for (IndexableField field : doc.getFields("double")) {
+            assertEquals(IndexOptions.NONE, field.fieldType().indexOptions());
+        }
+    }
+
+    public void testBwCompatIndex() throws IOException {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties")
+                .startObject("int")
+                    .field("type", "integer")
+                    .field("index", "no")
+                .endObject()
+                .startObject("double")
+                    .field("type", "double")
+                    .field("index", "not_analyzed")
+                .endObject()
+                .endObject()
+                .endObject().endObject().string();
+
+        Settings oldSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_2_0).build();
+        DocumentMapper defaultMapper = createIndex("test", oldSettings).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        assertEquals("{\"type\":{\"properties\":{\"double\":{\"type\":\"double\"},\"int\":{\"type\":\"integer\",\"index\":false}}}}",
+                defaultMapper.mapping().toString());
+    }
+
     public void testDocValuesOnNested() throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("properties")

core/src/test/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsIT.java

@@ -143,7 +143,7 @@ public class SimpleGetFieldMappingsIT extends ESIntegTestCase {
 
         GetFieldMappingsResponse response = client().admin().indices().prepareGetFieldMappings().setFields("num", "field1", "obj.subfield").includeDefaults(true).get();
 
-        assertThat((Map<String, Object>) response.fieldMappings("test", "type", "num").sourceAsMap().get("num"), hasEntry("index", (Object) "not_analyzed"));
+        assertThat((Map<String, Object>) response.fieldMappings("test", "type", "num").sourceAsMap().get("num"), hasEntry("index", Boolean.TRUE));
         assertThat((Map<String, Object>) response.fieldMappings("test", "type", "num").sourceAsMap().get("num"), hasEntry("type", (Object) "long"));
         assertThat((Map<String, Object>) response.fieldMappings("test", "type", "field1").sourceAsMap().get("field1"), hasEntry("index", (Object) "analyzed"));
         assertThat((Map<String, Object>) response.fieldMappings("test", "type", "field1").sourceAsMap().get("field1"), hasEntry("type", (Object) "string"));

core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsIT.java

@@ -76,7 +76,7 @@ public class SignificantTermsIT extends ESIntegTestCase {
     public void setupSuiteScopeCluster() throws Exception {
         assertAcked(prepareCreate("test").setSettings(SETTING_NUMBER_OF_SHARDS, 5, SETTING_NUMBER_OF_REPLICAS, 0).addMapping("fact",
                 "_routing", "required=true", "routing_id", "type=string,index=not_analyzed", "fact_category",
-                "type=integer,index=not_analyzed", "description", "type=string,index=analyzed"));
+                "type=integer,index=true", "description", "type=string,index=analyzed"));
         createIndex("idx_unmapped");
 
         ensureGreen();

core/src/test/resources/org/elasticsearch/index/mapper/multifield/test-multi-fields.json

@@ -26,7 +26,7 @@
           },
           "test2": {
             "type": "token_count",
-            "index": "not_analyzed",
+            "index": true,
             "store": true,
             "analyzer": "simple"
           }

docs/reference/mapping/types/boolean.asciidoc

@@ -104,7 +104,7 @@ The following parameters are accepted by `boolean` fields:
 
 <<mapping-index,`index`>>::
 
-    Should the field be searchable? Accepts `not_analyzed` (default) and `no`.
+    Should the field be searchable? Accepts `true` (default) and `false`.
 
 <<null-value,`null_value`>>::
 

docs/reference/mapping/types/date.asciidoc

@@ -115,13 +115,13 @@ The following parameters are accepted by `date` fields:
 
     Whether or not the field value should be included in the
     <<mapping-all-field,`_all`>> field? Accepts `true` or `false`.  Defaults
-    to `false` if <<mapping-index,`index`>> is set to `no`, or if a parent
+    to `false` if <<mapping-index,`index`>> is set to `false`, or if a parent
     <<object,`object`>> field sets `include_in_all` to `false`.
     Otherwise defaults to `true`.
 
 <<mapping-index,`index`>>::
 
-    Should the field be searchable? Accepts `not_analyzed` (default) and `no`.
+    Should the field be searchable? Accepts `true` (default) and `false`.
 
 <<null-value,`null_value`>>::
 

docs/reference/mapping/types/ip.asciidoc

@@ -62,13 +62,13 @@ The following parameters are accepted by `ip` fields:
 
     Whether or not the field value should be included in the
     <<mapping-all-field,`_all`>> field? Accepts `true` or `false`.  Defaults
-    to `false` if <<mapping-index,`index`>> is set to `no`, or if a parent
+    to `false` if <<mapping-index,`index`>> is set to `false`, or if a parent
     <<object,`object`>> field sets `include_in_all` to `false`.
     Otherwise defaults to `true`.
 
 <<mapping-index,`index`>>::
 
-    Should the field be searchable? Accepts `not_analyzed` (default) and `no`.
+    Should the field be searchable? Accepts `true` (default) and `false`.
 
 <<null-value,`null_value`>>::
 

docs/reference/mapping/types/numeric.asciidoc

@@ -65,13 +65,13 @@ The following parameters are accepted by numeric types:
 
     Whether or not the field value should be included in the
     <<mapping-all-field,`_all`>> field? Accepts `true` or `false`.  Defaults
-    to `false` if <<mapping-index,`index`>> is set to `no`, or if a parent
+    to `false` if <<mapping-index,`index`>> is set to `false`, or if a parent
     <<object,`object`>> field sets `include_in_all` to `false`.
     Otherwise defaults to `true`.
 
 <<mapping-index,`index`>>::
 
-    Should the field be searchable? Accepts `not_analyzed` (default) and `no`.
+    Should the field be searchable? Accepts `true` (default) and `false`.
 
 <<null-value,`null_value`>>::
 

docs/reference/migration/migrate_3_0.asciidoc

@@ -273,6 +273,12 @@ float by default instead of a double. The reasoning is that floats should be
 more than enough for most cases but would decrease storage requirements
 significantly.
 
+==== `index` property
+
+On all types but `string`, the `index` property now only accepts `true`/`false`
+instead of `not_analyzed`/`no`. The `string` field still accepts
+`analyzed`/`not_analyzed`/`no`.
+
 ==== `_source`'s `format` option
 
 The `_source` mapping does not support the `format` option anymore. This option