Share same existsQuery impl throughout mappers (#57607)

Most of our field types have the same implementation for their `existsQuery` method which relies on doc_values if present, otherwise it queries norms if available or uses a term query against the _field_names meta field. This standard implementation is repeated in many different mappers.

There are field types that only query doc_values, because they always have them, and field types that always query _field_names, because they never have norms nor doc_values. We could apply the same standard logic to all of these field types as `MappedFieldType` has the knowledge about what data structures are available.

This commit introduces a standard implementation that does the right thing depending on the data structure that is available. With that only field types that require a different behaviour need to override the existsQuery method.

At the same time, this no longer forces subclasses to override `existsQuery`, which could be forgotten when needed. To address this we introduced a new test method in `MapperTestCase` that verifies the `existsQuery` being generated and its consistency with the available data structures.

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java

@@ -24,11 +24,8 @@ import org.apache.lucene.index.DocValues;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.NumericDocValues;
 import org.apache.lucene.index.SortedNumericDocValues;
-import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BoostQuery;
-import org.apache.lucene.search.DocValuesFieldExistsQuery;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.settings.Setting;
@@ -156,15 +153,6 @@ public class ScaledFloatFieldMapper extends ParametrizedFieldMapper {
             return CONTENT_TYPE;
         }
 
-        @Override
-        public Query existsQuery(QueryShardContext context) {
-            if (hasDocValues()) {
-                return new DocValuesFieldExistsQuery(name());
-            } else {
-                return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name()));
-            }
-        }
-
         @Override
         public Query termQuery(Object value, QueryShardContext context) {
             failIfNotIndexed();

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java

@@ -36,7 +36,6 @@ import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.MultiTermQuery;
-import org.apache.lucene.search.NormsFieldExistsQuery;
 import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
@@ -271,15 +270,6 @@ public class SearchAsYouTypeFieldMapper extends FieldMapper {
             return shingleFields[Math.min(indexFromShingleSize, shingleFields.length - 1)];
         }
 
-        @Override
-        public Query existsQuery(QueryShardContext context) {
-            if (getTextSearchInfo().hasNorms() == false) {
-                return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name()));
-            } else {
-                return new NormsFieldExistsQuery(name());
-            }
-        }
-
         @Override
         public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, boolean caseInsensitive, QueryShardContext context) {
             if (prefixField == null || prefixField.termLengthWithinBounds(value.length()) == false) {
@@ -500,15 +490,6 @@ public class SearchAsYouTypeFieldMapper extends FieldMapper {
             return CONTENT_TYPE;
         }
 
-        @Override
-        public Query existsQuery(QueryShardContext context) {
-            if (getTextSearchInfo().hasNorms() == false) {
-                return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name()));
-            } else {
-                return new NormsFieldExistsQuery(name());
-            }
-        }
-
         @Override
         public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, boolean caseInsensitive, QueryShardContext context) {
             if (prefixFieldType == null || prefixFieldType.termLengthWithinBounds(value.length()) == false) {

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java

@@ -23,6 +23,8 @@ import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.TermFrequencyAttribute;
 import org.apache.lucene.document.FeatureField;
 import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.Strings;
@@ -38,7 +40,15 @@ import java.util.Collection;
 import java.util.List;
 import java.util.Set;
 
+import static org.hamcrest.Matchers.instanceOf;
+
 public class RankFeatureFieldMapperTests extends FieldMapperTestCase2<RankFeatureFieldMapper.Builder> {
+
+    @Override
+    protected void writeFieldValue(XContentBuilder builder) throws IOException {
+        builder.value(10);
+    }
+
     @Override
     protected Set<String> unsupportedProperties() {
         return Set.of("analyzer", "similarity", "store", "doc_values", "index");
@@ -52,6 +62,15 @@ public class RankFeatureFieldMapperTests extends FieldMapperTestCase2<RankFeatur
         });
     }
 
+    @Override
+    protected void assertExistsQuery(MappedFieldType fieldType, Query query, ParseContext.Document fields) {
+        assertThat(query, instanceOf(TermQuery.class));
+        TermQuery termQuery = (TermQuery) query;
+        assertEquals("_feature", termQuery.getTerm().field());
+        assertEquals("field", termQuery.getTerm().text());
+        assertNotNull(fields.getField("_feature"));
+    }
+
     @Override
     protected Collection<? extends Plugin> getPlugins() {
         return List.of(new MapperExtrasPlugin());

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapperTests.java

@@ -34,6 +34,17 @@ import java.util.Set;
 
 public class RankFeaturesFieldMapperTests extends FieldMapperTestCase2<RankFeaturesFieldMapper.Builder> {
 
+    @Override
+    protected void writeFieldValue(XContentBuilder builder) throws IOException {
+        builder.startObject().field("foo", 10).field("bar", 20).endObject();
+    }
+
+    @Override
+    protected void assertExistsQuery(MapperService mapperService) {
+        IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> super.assertExistsQuery(mapperService));
+        assertEquals("[rank_features] fields do not support [exists] queries", iae.getMessage());
+    }
+
     @Override
     protected Set<String> unsupportedProperties() {
         return Set.of("analyzer", "similarity", "store", "doc_values", "index");
@@ -58,7 +69,7 @@ public class RankFeaturesFieldMapperTests extends FieldMapperTestCase2<RankFeatu
         DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping));
         assertEquals(Strings.toString(fieldMapping(this::minimalMapping)), mapper.mappingSource().toString());
 
-        ParsedDocument doc1 = mapper.parse(source(b -> b.startObject("field").field("foo", 10).field("bar", 20).endObject()));
+        ParsedDocument doc1 = mapper.parse(source(this::writeField));
 
         IndexableField[] fields = doc1.rootDoc().getFields("field");
         assertEquals(2, fields.length);

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java

@@ -37,7 +37,6 @@ import java.util.Collection;
 import java.util.List;
 
 import static java.util.Collections.singletonList;
-import static org.elasticsearch.index.mapper.FieldMapperTestCase.fetchSourceValue;
 import static org.hamcrest.Matchers.containsString;
 
 public class ScaledFloatFieldMapperTests extends MapperTestCase {
@@ -47,23 +46,31 @@ public class ScaledFloatFieldMapperTests extends MapperTestCase {
         return singletonList(new MapperExtrasPlugin());
     }
 
+    @Override
+    protected void writeFieldValue(XContentBuilder builder) throws IOException {
+        builder.value(123);
+    }
+
     @Override
     protected void minimalMapping(XContentBuilder b) throws IOException {
         b.field("type", "scaled_float").field("scaling_factor", 10.0);
     }
 
+    public void testExistsQueryDocValuesDisabled() throws IOException {
+        MapperService mapperService = createMapperService(fieldMapping(b -> {
+            minimalMapping(b);
+            b.field("doc_values", false);
+        }));
+        assertExistsQuery(mapperService);
+        assertParseMinimalWarnings();
+    }
+
     public void testDefaults() throws Exception {
         XContentBuilder mapping = fieldMapping(b -> b.field("type", "scaled_float").field("scaling_factor", 10.0));
         DocumentMapper mapper = createDocumentMapper(mapping);
         assertEquals(Strings.toString(mapping), mapper.mappingSource().toString());
 
-        ParsedDocument doc = mapper.parse(new SourceToParse("test", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
-                        .startObject()
-                        .field("field", 123)
-                        .endObject()),
-                XContentType.JSON));
-
+        ParsedDocument doc = mapper.parse(source(b -> b.field("field", 123)));
         IndexableField[] fields = doc.rootDoc().getFields("field");
         assertEquals(2, fields.length);
         IndexableField pointField = fields[0];

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java

@@ -80,6 +80,11 @@ import static org.hamcrest.core.IsInstanceOf.instanceOf;
 
 public class SearchAsYouTypeFieldMapperTests extends FieldMapperTestCase2<SearchAsYouTypeFieldMapper.Builder> {
 
+    @Override
+    protected void writeFieldValue(XContentBuilder builder) throws IOException {
+        builder.value("new york city");
+    }
+
     @Before
     public void addModifiers() {
         addModifier("max_shingle_size", false, (a, b) -> {
@@ -171,7 +176,7 @@ public class SearchAsYouTypeFieldMapperTests extends FieldMapperTestCase2<Search
             fieldMapping(
                 b -> b.field("type", "search_as_you_type").field("analyzer", analyzerName).field("max_shingle_size", maxShingleSize)
             )
-        ); 
+        );
 
         SearchAsYouTypeFieldMapper rootMapper = getRootFieldMapper(defaultMapper, "field");
         assertRootFieldMapper(rootMapper, maxShingleSize, analyzerName);

modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java

@@ -81,7 +81,7 @@ public class MetaJoinFieldMapper extends FieldMapper {
 
         private final String joinField;
 
-        MetaJoinFieldType(String joinField) {
+        private MetaJoinFieldType(String joinField) {
             super(NAME, false, false, false, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap());
             this.joinField = joinField;
         }

modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java

@@ -27,7 +27,6 @@ import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
-import org.apache.lucene.search.DocValuesFieldExistsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.BytesRef;
@@ -41,7 +40,6 @@ import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.StringFieldType;
 import org.elasticsearch.index.mapper.TextSearchInfo;
 import org.elasticsearch.index.mapper.ValueFetcher;
-import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
 import org.elasticsearch.search.lookup.SearchLookup;
 
@@ -98,7 +96,7 @@ public final class ParentIdFieldMapper extends FieldMapper {
     }
 
     public static final class ParentIdFieldType extends StringFieldType {
-        ParentIdFieldType(String name, boolean eagerGlobalOrdinals, Map<String, String> meta) {
+        private ParentIdFieldType(String name, boolean eagerGlobalOrdinals, Map<String, String> meta) {
             super(name, true, false, true, TextSearchInfo.SIMPLE_MATCH_ONLY, meta);
             setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
             setEagerGlobalOrdinals(eagerGlobalOrdinals);
@@ -123,11 +121,6 @@ public final class ParentIdFieldMapper extends FieldMapper {
             BytesRef binaryValue = (BytesRef) value;
             return binaryValue.utf8ToString();
         }
-
-        @Override
-        public Query existsQuery(QueryShardContext context) {
-            return new DocValuesFieldExistsQuery(name());
-        }
     }
 
     private final String parentName;

modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java

@@ -23,8 +23,6 @@ import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.SortedDocValuesField;
 import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.search.DocValuesFieldExistsQuery;
-import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -46,7 +44,6 @@ import org.elasticsearch.index.mapper.SourceValueFetcher;
 import org.elasticsearch.index.mapper.StringFieldType;
 import org.elasticsearch.index.mapper.TextSearchInfo;
 import org.elasticsearch.index.mapper.ValueFetcher;
-import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
 import org.elasticsearch.search.lookup.SearchLookup;
 
@@ -209,7 +206,7 @@ public final class ParentJoinFieldMapper extends FieldMapper {
     }
 
     public static final class JoinFieldType extends StringFieldType {
-        public JoinFieldType(String name, Map<String, String> meta) {
+        private JoinFieldType(String name, Map<String, String> meta) {
             super(name, true, false, true, TextSearchInfo.SIMPLE_MATCH_ONLY, meta);
             setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
         }
@@ -233,11 +230,6 @@ public final class ParentJoinFieldMapper extends FieldMapper {
             BytesRef binaryValue = (BytesRef) value;
             return binaryValue.utf8ToString();
         }
-
-        @Override
-        public Query existsQuery(QueryShardContext context) {
-            return new DocValuesFieldExistsQuery(name());
-        }
     }
 
     // The meta field that ensures that there is no other parent-join in the mapping

modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java

@@ -33,7 +33,6 @@ import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.CoveringQuery;
-import org.apache.lucene.search.DocValuesFieldExistsQuery;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.LongValuesSource;
 import org.apache.lucene.search.MatchNoDocsQuery;
@@ -57,7 +56,6 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.mapper.BinaryFieldMapper;
 import org.elasticsearch.index.mapper.FieldMapper;
-import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
@@ -194,7 +192,7 @@ public class PercolatorFieldMapper extends FieldMapper {
         RangeFieldMapper.RangeFieldType rangeField;
         boolean mapUnmappedFieldsAsText;
 
-        PercolatorFieldType(String name, Map<String, String> meta) {
+        private PercolatorFieldType(String name, Map<String, String> meta) {
             super(name, false, false, false, TextSearchInfo.NONE, meta);
         }
 
@@ -203,15 +201,6 @@ public class PercolatorFieldMapper extends FieldMapper {
             return CONTENT_TYPE;
         }
 
-        @Override
-        public Query existsQuery(QueryShardContext context) {
-            if (hasDocValues()) {
-                return new DocValuesFieldExistsQuery(name());
-            } else {
-                return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name()));
-            }
-        }
-
         @Override
         public Query termQuery(Object value, QueryShardContext context) {
             throw new QueryShardException(context, "Percolator fields are not searchable directly, use a percolate query instead");

plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java

@@ -27,11 +27,8 @@ import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.DocValuesFieldExistsQuery;
 import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -98,15 +95,6 @@ public class ICUCollationKeywordFieldMapper extends FieldMapper {
             return collator;
         }
 
-        @Override
-        public Query existsQuery(QueryShardContext context) {
-            if (hasDocValues()) {
-                return new DocValuesFieldExistsQuery(name());
-            } else {
-                return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name()));
-            }
-        }
-
         @Override
         public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier<SearchLookup> searchLookup) {
             failIfNoDocValues();

plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java

@@ -21,7 +21,6 @@ package org.elasticsearch.index.mapper;
 import com.ibm.icu.text.Collator;
 import com.ibm.icu.text.RawCollationKey;
 import com.ibm.icu.util.ULocale;
-
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexableField;
@@ -99,6 +98,11 @@ public class ICUCollationKeywordFieldMapperTests extends FieldMapperTestCase2<IC
         b.field("type", FIELD_TYPE);
     }
 
+    @Override
+    protected void writeFieldValue(XContentBuilder builder) throws IOException {
+        builder.value(1234);
+    }
+
     public void testDefaults() throws Exception {
         DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping));
         assertEquals(Strings.toString(fieldMapping(this::minimalMapping)), mapper.mappingSource().toString());

plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java

@@ -516,8 +516,8 @@ public class AnnotatedTextFieldMapper extends FieldMapper {
 
     public static final class AnnotatedTextFieldType extends TextFieldMapper.TextFieldType {
 
-        public AnnotatedTextFieldType(String name, FieldType fieldType, SimilarityProvider similarity,
-            NamedAnalyzer searchAnalyzer, NamedAnalyzer searchQuoteAnalyzer, Map<String, String> meta) {
+        private AnnotatedTextFieldType(String name, FieldType fieldType, SimilarityProvider similarity,
+                                       NamedAnalyzer searchAnalyzer, NamedAnalyzer searchQuoteAnalyzer, Map<String, String> meta) {
             super(name, fieldType, similarity, searchAnalyzer, searchQuoteAnalyzer, meta);
         }
 

plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java

@@ -23,7 +23,6 @@ import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.search.DocValuesFieldExistsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.hash.MurmurHash3;
@@ -97,7 +96,7 @@ public class Murmur3FieldMapper extends FieldMapper {
 
     // this only exists so a check can be done to match the field type to using murmur3 hashing...
     public static class Murmur3FieldType extends MappedFieldType {
-        public Murmur3FieldType(String name, boolean isStored, Map<String, String> meta) {
+        private Murmur3FieldType(String name, boolean isStored, Map<String, String> meta) {
             super(name, false, isStored, true, TextSearchInfo.SIMPLE_MATCH_ONLY, meta);
         }
 
@@ -112,11 +111,6 @@ public class Murmur3FieldMapper extends FieldMapper {
             return new SortedNumericIndexFieldData.Builder(name(), NumericType.LONG);
         }
 
-        @Override
-        public Query existsQuery(QueryShardContext context) {
-            return new DocValuesFieldExistsQuery(name());
-        }
-
         @Override
         public Query termQuery(Object value, QueryShardContext context) {
             throw new QueryShardException(context, "Murmur3 fields are not searchable: [" + name() + "]");

plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java

@@ -39,6 +39,12 @@ import java.util.Set;
 import static org.hamcrest.Matchers.containsString;
 
 public class Murmur3FieldMapperTests extends FieldMapperTestCase2<Murmur3FieldMapper.Builder> {
+
+    @Override
+    protected void writeFieldValue(XContentBuilder builder) throws IOException {
+        builder.value("value");
+    }
+
     @Override
     protected Set<String> unsupportedProperties() {
         return Set.of("analyzer", "similarity", "doc_values", "index");

server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java

@@ -21,10 +21,7 @@ package org.elasticsearch.index.mapper;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.DocValuesFieldExistsQuery;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.geo.GeoJsonGeometryFormat;
@@ -284,15 +281,6 @@ public abstract class AbstractGeometryFieldMapper<Parsed, Processed> extends Fie
             }
         }
 
-        @Override
-        public Query existsQuery(QueryShardContext context) {
-            if (hasDocValues()) {
-                return new DocValuesFieldExistsQuery(name());
-            } else {
-                return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name()));
-            }
-        }
-
         @Override
         public Query termQuery(Object value, QueryShardContext context) {
             throw new QueryShardException(context,

server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java

@@ -21,10 +21,7 @@ package org.elasticsearch.index.mapper;
 
 import com.carrotsearch.hppc.ObjectArrayList;
 import org.apache.lucene.document.StoredField;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.DocValuesFieldExistsQuery;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.store.ByteArrayDataOutput;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ElasticsearchException;
@@ -87,7 +84,7 @@ public class BinaryFieldMapper extends ParametrizedFieldMapper {
 
     public static final class BinaryFieldType extends MappedFieldType {
 
-        public BinaryFieldType(String name, boolean isStored, boolean hasDocValues, Map<String, String> meta) {
+        private BinaryFieldType(String name, boolean isStored, boolean hasDocValues, Map<String, String> meta) {
             super(name, false, isStored, hasDocValues, TextSearchInfo.NONE, meta);
         }
 
@@ -130,15 +127,6 @@ public class BinaryFieldMapper extends ParametrizedFieldMapper {
             return new BytesBinaryIndexFieldData.Builder(name(), CoreValuesSourceType.BYTES);
         }
 
-        @Override
-        public Query existsQuery(QueryShardContext context) {
-            if (hasDocValues()) {
-                return new DocValuesFieldExistsQuery(name());
-            } else {
-                return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name()));
-            }
-        }
-
         @Override
         public Query termQuery(Object value, QueryShardContext context) {
             throw new QueryShardException(context, "Binary fields do not support searching");

server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java

@@ -24,10 +24,7 @@ import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.DocValuesFieldExistsQuery;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TermRangeQuery;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.Booleans;
@@ -124,15 +121,6 @@ public class BooleanFieldMapper extends ParametrizedFieldMapper {
             return CONTENT_TYPE;
         }
 
-        @Override
-        public Query existsQuery(QueryShardContext context) {
-            if (hasDocValues()) {
-                return new DocValuesFieldExistsQuery(name());
-            } else {
-                return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name()));
-            }
-        }
-
         @Override
         public BytesRef indexedValueForSearch(Object value) {
             if (value == null) {

server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java

@@ -22,8 +22,6 @@ import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.suggest.document.Completion84PostingsFormat;
 import org.apache.lucene.search.suggest.document.CompletionAnalyzer;
 import org.apache.lucene.search.suggest.document.CompletionQuery;
@@ -44,7 +42,6 @@ import org.elasticsearch.common.xcontent.XContentParser.NumberType;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
 import org.elasticsearch.index.analysis.AnalyzerScope;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
-import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.search.suggest.completion.CompletionSuggester;
 import org.elasticsearch.search.suggest.completion.context.ContextMapping;
@@ -292,11 +289,6 @@ public class CompletionFieldMapper extends ParametrizedFieldMapper {
             return postingsFormat;
         }
 
-        @Override
-        public Query existsQuery(QueryShardContext context) {
-            return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name()));
-        }
-
         /**
          * Completion prefix query
          */

server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java

@@ -24,13 +24,10 @@ import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.PointValues;
-import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BoostQuery;
-import org.apache.lucene.search.DocValuesFieldExistsQuery;
 import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.Nullable;
@@ -317,15 +314,6 @@ public final class DateFieldMapper extends ParametrizedFieldMapper {
             return resolution.convert(DateFormatters.from(dateTimeFormatter().parse(value), dateTimeFormatter().locale()).toInstant());
         }
 
-        @Override
-        public Query existsQuery(QueryShardContext context) {
-            if (hasDocValues()) {
-                return new DocValuesFieldExistsQuery(name());
-            } else {
-                return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name()));
-            }
-        }
-
         @Override
         public Query termQuery(Object value, @Nullable QueryShardContext context) {
             Query query = rangeQuery(value, value, true, true, ShapeRelation.INTERSECTS, null, null, context);

server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java

@@ -21,7 +21,6 @@ package org.elasticsearch.index.mapper;
 
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
-
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.index.IndexOptions;
@@ -282,7 +281,8 @@ public abstract class FieldMapper extends Mapper implements Cloneable {
      */
     public abstract ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, @Nullable String format);
 
-    protected void createFieldNamesField(ParseContext context) {
+    protected final void createFieldNamesField(ParseContext context) {
+        assert fieldType().hasDocValues() == false : "_field_names should only be used when doc_values are turned off";
         FieldNamesFieldType fieldNamesFieldType = context.docMapper().metadataMapper(FieldNamesFieldMapper.class).fieldType();
         if (fieldNamesFieldType != null && fieldNamesFieldType.isEnabled()) {
             for (String fieldName : FieldNamesFieldMapper.extractFieldNames(fieldType().name())) {

server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java

@@ -172,7 +172,7 @@ public class GeoPointFieldMapper extends AbstractPointGeometryFieldMapper<List<P
     }
 
     public static class GeoPointFieldType extends AbstractPointGeometryFieldType<List<ParsedGeoPoint>, List<? extends GeoPoint>> {
-        public GeoPointFieldType(String name, boolean indexed, boolean stored, boolean hasDocValues, Map<String, String> meta) {
+        private GeoPointFieldType(String name, boolean indexed, boolean stored, boolean hasDocValues, Map<String, String> meta) {
             super(name, indexed, stored, hasDocValues, meta);
         }
 

server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java

@@ -23,11 +23,8 @@ import org.apache.lucene.document.InetAddressPoint;
 import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.index.SortedSetDocValues;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.DocValuesFieldExistsQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.Version;
@@ -155,15 +152,6 @@ public class IpFieldMapper extends ParametrizedFieldMapper {
             }
         }
 
-        @Override
-        public Query existsQuery(QueryShardContext context) {
-            if (hasDocValues()) {
-                return new DocValuesFieldExistsQuery(name());
-            } else {
-                return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name()));
-            }
-        }
-
         @Override
         public Query termQuery(Object value, @Nullable QueryShardContext context) {
             failIfNotIndexed();

server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java

@@ -25,11 +25,6 @@ import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.DocValuesFieldExistsQuery;
-import org.apache.lucene.search.NormsFieldExistsQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -37,7 +32,6 @@ import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData;
-import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.similarity.SimilarityProvider;
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
 import org.elasticsearch.search.lookup.SearchLookup;
@@ -202,14 +196,11 @@ public final class KeywordFieldMapper extends ParametrizedFieldMapper {
 
     public static final class KeywordFieldType extends StringFieldType {
 
-        boolean hasNorms;
-
         public KeywordFieldType(String name, boolean hasDocValues, FieldType fieldType,
                                 boolean eagerGlobalOrdinals, NamedAnalyzer normalizer, NamedAnalyzer searchAnalyzer,
                                 SimilarityProvider similarity, float boost, Map<String, String> meta) {
             super(name, fieldType.indexOptions() != IndexOptions.NONE, fieldType.stored(),
                 hasDocValues, new TextSearchInfo(fieldType, similarity, searchAnalyzer, searchAnalyzer), meta);
-            this.hasNorms = fieldType.omitNorms() == false;
             setEagerGlobalOrdinals(eagerGlobalOrdinals);
             setIndexAnalyzer(normalizer);
             setBoost(boost);
@@ -238,17 +229,6 @@ public final class KeywordFieldMapper extends ParametrizedFieldMapper {
             return indexAnalyzer();
         }
 
-        @Override
-        public Query existsQuery(QueryShardContext context) {
-            if (hasDocValues()) {
-                return new DocValuesFieldExistsQuery(name());
-            } else if (hasNorms == false) {
-                return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name()));
-            } else {
-                return new NormsFieldExistsQuery(name());
-            }
-        }
-
         @Override
         public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier<SearchLookup> searchLookup) {
             failIfNoDocValues();

server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java

@@ -322,7 +322,7 @@ public class LegacyGeoShapeFieldMapper extends AbstractShapeGeometryFieldMapper<
         private RecursivePrefixTreeStrategy recursiveStrategy;
         private TermQueryPrefixTreeStrategy termStrategy;
 
-        public GeoShapeFieldType(String name, boolean indexed, boolean stored, boolean hasDocValues, Map<String, String> meta) {
+        private GeoShapeFieldType(String name, boolean indexed, boolean stored, boolean hasDocValues, Map<String, String> meta) {
             super(name, indexed, stored, hasDocValues, meta);
         }
 

server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java

@@ -29,7 +29,9 @@ import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
+import org.apache.lucene.search.DocValuesFieldExistsQuery;
 import org.apache.lucene.search.MultiTermQuery;
+import org.apache.lucene.search.NormsFieldExistsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermInSetQuery;
 import org.apache.lucene.search.TermQuery;
@@ -254,7 +256,15 @@ public abstract class MappedFieldType {
             + "] which is of type [" + typeName() + "]");
     }
 
-    public abstract Query existsQuery(QueryShardContext context);
+    public Query existsQuery(QueryShardContext context) {
+        if (hasDocValues()) {
+            return new DocValuesFieldExistsQuery(name());
+        } else if (getTextSearchInfo().hasNorms()) {
+            return new NormsFieldExistsQuery(name());
+        } else {
+            return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name()));
+        }
+    }
 
     public Query phraseQuery(TokenStream stream, int slop, boolean enablePositionIncrements) throws IOException {
         throw new IllegalArgumentException("Can only use phrase queries on text fields - not on [" + name

server/src/main/java/org/elasticsearch/index/mapper/NestedPathFieldMapper.java

@@ -73,7 +73,7 @@ public class NestedPathFieldMapper extends MetadataFieldMapper {
 
     public static final class NestedPathFieldType extends StringFieldType {
 
-        NestedPathFieldType(Settings settings) {
+        private NestedPathFieldType(Settings settings) {
             super(NestedPathFieldMapper.name(settings), true, false, false, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap());
         }
 

server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java

@@ -29,14 +29,11 @@ import org.apache.lucene.document.IntPoint;
 import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.document.StoredField;
-import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BoostQuery;
-import org.apache.lucene.search.DocValuesFieldExistsQuery;
 import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.NumericUtils;
 import org.elasticsearch.common.Explicit;
@@ -917,15 +914,6 @@ public class NumberFieldMapper extends ParametrizedFieldMapper {
             return type.numericType();
         }
 
-        @Override
-        public Query existsQuery(QueryShardContext context) {
-            if (hasDocValues()) {
-                return new DocValuesFieldExistsQuery(name());
-            } else {
-                return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name()));
-            }
-        }
-
         @Override
         public Query termQuery(Object value, QueryShardContext context) {
             failIfNotIndexed();

server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java

@@ -19,11 +19,8 @@
 
 package org.elasticsearch.index.mapper;
 
-import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BoostQuery;
-import org.apache.lucene.search.DocValuesFieldExistsQuery;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.Explicit;
@@ -203,15 +200,6 @@ public class RangeFieldMapper extends ParametrizedFieldMapper {
             return dateMathParser;
         }
 
-        @Override
-        public Query existsQuery(QueryShardContext context) {
-            if (hasDocValues()) {
-                return new DocValuesFieldExistsQuery(name());
-            } else {
-                return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name()));
-            }
-        }
-
         @Override
         public DocValueFormat docValueFormat(String format, ZoneId timeZone) {
             if (rangeType == RangeType.DATE) {

server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java

@@ -22,11 +22,7 @@ package org.elasticsearch.index.mapper;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
 import org.elasticsearch.common.lucene.Lucene;
-import org.elasticsearch.index.query.QueryShardContext;
 
 import java.util.Collections;
 import java.util.List;
@@ -96,11 +92,6 @@ public class RoutingFieldMapper extends MetadataFieldMapper {
         public String typeName() {
             return CONTENT_TYPE;
         }
-
-        @Override
-        public Query existsQuery(QueryShardContext context) {
-            return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name()));
-        }
     }
 
     private final boolean required;

server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java

@@ -22,7 +22,6 @@ package org.elasticsearch.index.mapper;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.document.NumericDocValuesField;
-import org.apache.lucene.search.DocValuesFieldExistsQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
@@ -96,7 +95,7 @@ public class SeqNoFieldMapper extends MetadataFieldMapper {
 
         private static final SeqNoFieldType INSTANCE = new SeqNoFieldType();
 
-        SeqNoFieldType() {
+        private SeqNoFieldType() {
             super(NAME, true, false, true, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap());
         }
 
@@ -122,11 +121,6 @@ public class SeqNoFieldMapper extends MetadataFieldMapper {
             return Long.parseLong(value.toString());
         }
 
-        @Override
-        public Query existsQuery(QueryShardContext context) {
-            return new DocValuesFieldExistsQuery(name());
-        }
-
         @Override
         public Query termQuery(Object value, @Nullable QueryShardContext context) {
             long v = parse(value);

server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java

@@ -41,7 +41,6 @@ import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.MultiPhraseQuery;
 import org.apache.lucene.search.MultiTermQuery;
-import org.apache.lucene.search.NormsFieldExistsQuery;
 import org.apache.lucene.search.PhraseQuery;
 import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
@@ -380,7 +379,7 @@ public class TextFieldMapper extends FieldMapper {
 
         final TextFieldType parent;
 
-        PhraseFieldType(TextFieldType parent) {
+        private PhraseFieldType(TextFieldType parent) {
             super(parent.name() + FAST_PHRASE_SUFFIX, true, false, false, parent.getTextSearchInfo(), Collections.emptyMap());
             setAnalyzer(parent.indexAnalyzer().name(), parent.indexAnalyzer().analyzer());
             this.parent = parent;
@@ -567,13 +566,11 @@ public class TextFieldMapper extends FieldMapper {
         private int fielddataMinSegmentSize;
         private PrefixFieldType prefixFieldType;
         private boolean indexPhrases = false;
-        private final FieldType indexedFieldType;
 
         public TextFieldType(String name, FieldType indexedFieldType, SimilarityProvider similarity, NamedAnalyzer searchAnalyzer,
                              NamedAnalyzer searchQuoteAnalyzer, Map<String, String> meta) {
             super(name, indexedFieldType.indexOptions() != IndexOptions.NONE, indexedFieldType.stored(), false,
                 new TextSearchInfo(indexedFieldType, similarity, searchAnalyzer, searchQuoteAnalyzer), meta);
-            this.indexedFieldType = indexedFieldType;
             fielddata = false;
             fielddataMinFrequency = Defaults.FIELDDATA_MIN_FREQUENCY;
             fielddataMaxFrequency = Defaults.FIELDDATA_MAX_FREQUENCY;
@@ -583,7 +580,6 @@ public class TextFieldMapper extends FieldMapper {
         public TextFieldType(String name, boolean indexed, boolean stored, Map<String, String> meta) {
             super(name, indexed, stored, false,
                 new TextSearchInfo(Defaults.FIELD_TYPE, null, Lucene.STANDARD_ANALYZER, Lucene.STANDARD_ANALYZER), meta);
-            this.indexedFieldType = Defaults.FIELD_TYPE;
             fielddata = false;
         }
 
@@ -670,15 +666,6 @@ public class TextFieldMapper extends FieldMapper {
             }
         }
 
-        @Override
-        public Query existsQuery(QueryShardContext context) {
-            if (indexedFieldType.omitNorms()) {
-                return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name()));
-            } else {
-                return new NormsFieldExistsQuery(name());
-            }
-        }
-
         @Override
         public IntervalsSource intervals(String text, int maxGaps, boolean ordered,
                                          NamedAnalyzer analyzer, boolean prefix) throws IOException {

server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java

@@ -21,7 +21,6 @@ package org.elasticsearch.index.mapper;
 
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.NumericDocValuesField;
-import org.apache.lucene.search.DocValuesFieldExistsQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.index.mapper.ParseContext.Document;
 import org.elasticsearch.index.query.QueryShardContext;
@@ -50,11 +49,6 @@ public class VersionFieldMapper extends MetadataFieldMapper {
             return CONTENT_TYPE;
         }
 
-        @Override
-        public Query existsQuery(QueryShardContext context) {
-            return new DocValuesFieldExistsQuery(name());
-        }
-
         @Override
         public Query termQuery(Object value, QueryShardContext context) {
             throw new QueryShardException(context, "The _version field is not searchable");

server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java

@@ -35,11 +35,50 @@ import static org.hamcrest.Matchers.instanceOf;
 
 public class BinaryFieldMapperTests extends MapperTestCase {
 
+    @Override
+    protected void writeFieldValue(XContentBuilder builder) throws IOException {
+        final byte[] binaryValue = new byte[100];
+        binaryValue[56] = 1;
+        builder.value(binaryValue);
+    }
+
     @Override
     protected void minimalMapping(XContentBuilder b) throws IOException {
         b.field("type", "binary");
     }
 
+    public void testExistsQueryDocValuesEnabled() throws IOException {
+        MapperService mapperService = createMapperService(fieldMapping(b -> {
+            minimalMapping(b);
+            b.field("doc_values", true);
+            if (randomBoolean()) {
+                b.field("store", randomBoolean());
+            }
+        }));
+        assertExistsQuery(mapperService);
+        assertParseMinimalWarnings();
+    }
+
+    public void testExistsQueryStoreEnabled() throws IOException {
+        MapperService mapperService = createMapperService(fieldMapping(b -> {
+            minimalMapping(b);
+            b.field("store", true);
+            if (randomBoolean()) {
+                b.field("doc_values", false);
+            }
+        }));
+        assertExistsQuery(mapperService);
+    }
+
+    public void testExistsQueryStoreAndDocValuesDiabled() throws IOException {
+        MapperService mapperService = createMapperService(fieldMapping(b -> {
+            minimalMapping(b);
+            b.field("store", false);
+            b.field("doc_values", false);
+        }));
+        assertExistsQuery(mapperService);
+    }
+
     public void testDefaultMapping() throws Exception {
         MapperService mapperService = createMapperService(fieldMapping(this::minimalMapping));
         FieldMapper mapper = (FieldMapper) mapperService.documentMapper().mappers().getMapper("field");

server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java

@@ -43,6 +43,11 @@ import java.util.Map;
 
 public class BooleanFieldMapperTests extends MapperTestCase {
 
+    @Override
+    protected void writeFieldValue(XContentBuilder builder) throws IOException {
+        builder.value(true);
+    }
+
     @Override
     protected void minimalMapping(XContentBuilder b) throws IOException {
         b.field("type", "boolean");
@@ -53,10 +58,18 @@ public class BooleanFieldMapperTests extends MapperTestCase {
         assertWarnings("Parameter [boost] on field [field] is deprecated and will be removed in 8.0");
     }
 
-    public void testDefaults() throws IOException {
+    public void testExistsQueryDocValuesDisabled() throws IOException {
+        MapperService mapperService = createMapperService(fieldMapping(b -> {
+            minimalMapping(b);
+            b.field("doc_values", false);
+        }));
+        assertExistsQuery(mapperService);
+        assertParseMinimalWarnings();
+    }
 
+    public void testDefaults() throws IOException {
         MapperService mapperService = createMapperService(fieldMapping(this::minimalMapping));
-        ParsedDocument doc = mapperService.documentMapper().parse(source(b -> b.field("field", true)));
+        ParsedDocument doc = mapperService.documentMapper().parse(source(this::writeField));
 
         withLuceneIndex(mapperService, iw -> iw.addDocument(doc.rootDoc()), reader -> {
             final LeafReader leaf = reader.leaves().get(0).reader();

server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java

@@ -67,6 +67,11 @@ import static org.hamcrest.Matchers.is;
 
 public class CompletionFieldMapperTests extends MapperTestCase {
 
+    @Override
+    protected void writeFieldValue(XContentBuilder builder) throws IOException {
+        builder.value("value");
+    }
+
     @Override
     protected void minimalMapping(XContentBuilder b) throws IOException {
         b.field("type", "completion");

server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java

@@ -45,11 +45,25 @@ import static org.hamcrest.Matchers.notNullValue;
 
 public class DateFieldMapperTests extends MapperTestCase {
 
+    @Override
+    protected void writeFieldValue(XContentBuilder builder) throws IOException {
+        builder.value("2016-03-11");
+    }
+
     @Override
     protected void minimalMapping(XContentBuilder b) throws IOException {
         b.field("type", "date");
     }
 
+    public void testExistsQueryDocValuesDisabled() throws IOException {
+        MapperService mapperService = createMapperService(fieldMapping(b -> {
+            minimalMapping(b);
+            b.field("doc_values", false);
+        }));
+        assertExistsQuery(mapperService);
+        assertParseMinimalWarnings();
+    }
+
     @Override
     protected void assertParseMaximalWarnings() {
         assertWarnings("Parameter [boost] on field [field] is deprecated and will be removed in 8.0");

server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java

@@ -24,14 +24,9 @@ import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.document.FieldType;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.DocValuesFieldExistsQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.index.analysis.AnalyzerScope;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
-import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.io.IOException;
@@ -73,7 +68,7 @@ public class DocumentFieldMapperTests extends LuceneTestCase {
 
     static class FakeFieldType extends TermBasedFieldType {
 
-        FakeFieldType(String name) {
+        private FakeFieldType(String name) {
             super(name, true, false, true, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap());
         }
 
@@ -81,16 +76,6 @@ public class DocumentFieldMapperTests extends LuceneTestCase {
         public String typeName() {
             return "fake";
         }
-
-        @Override
-        public Query existsQuery(QueryShardContext context) {
-            if (hasDocValues()) {
-                return new DocValuesFieldExistsQuery(name());
-            } else {
-                return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name()));
-            }
-        }
-
     }
 
     static class FakeFieldMapper extends FieldMapper {

server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java

@@ -20,16 +20,11 @@
 package org.elasticsearch.index.mapper;
 
 import org.apache.lucene.document.FieldType;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.DocValuesFieldExistsQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
 import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.builders.PointBuilder;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.geometry.Point;
-import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.io.IOException;
@@ -122,7 +117,7 @@ public class ExternalMapper extends FieldMapper {
 
     static class ExternalFieldType extends TermBasedFieldType {
 
-        ExternalFieldType(String name, boolean indexed, boolean stored, boolean hasDocValues) {
+        private ExternalFieldType(String name, boolean indexed, boolean stored, boolean hasDocValues) {
             super(name, indexed, stored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap());
         }
 
@@ -130,15 +125,6 @@ public class ExternalMapper extends FieldMapper {
         public String typeName() {
             return "faketype";
         }
-
-        @Override
-        public Query existsQuery(QueryShardContext context) {
-            if (hasDocValues()) {
-                return new DocValuesFieldExistsQuery(name());
-            } else {
-                return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name()));
-            }
-        }
     }
 
     private final String generatedValue;

server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java

@@ -23,14 +23,9 @@ import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.DocValuesFieldExistsQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.io.IOException;
@@ -88,7 +83,7 @@ public class FakeStringFieldMapper extends FieldMapper {
 
     public static final class FakeStringFieldType extends StringFieldType {
 
-        public FakeStringFieldType(String name, boolean stored, TextSearchInfo textSearchInfo) {
+        private FakeStringFieldType(String name, boolean stored, TextSearchInfo textSearchInfo) {
             super(name, true, stored, true, textSearchInfo, Collections.emptyMap());
             setIndexAnalyzer(Lucene.STANDARD_ANALYZER);
         }
@@ -97,15 +92,6 @@ public class FakeStringFieldMapper extends FieldMapper {
         public String typeName() {
             return CONTENT_TYPE;
         }
-
-        @Override
-        public Query existsQuery(QueryShardContext context) {
-            if (hasDocValues()) {
-                return new DocValuesFieldExistsQuery(name());
-            } else {
-                return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name()));
-            }
-        }
     }
 
     protected FakeStringFieldMapper(FieldType fieldType, MappedFieldType mappedFieldType,

server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java

@@ -56,6 +56,20 @@ public class GeoPointFieldMapperTests extends FieldMapperTestCase2<GeoPointField
         b.field("type", "geo_point");
     }
 
+    @Override
+    protected void writeFieldValue(XContentBuilder builder) throws IOException {
+        builder.value(stringEncode(1.3, 1.2));
+    }
+
+    public final void testExistsQueryDocValuesDisabled() throws IOException {
+        MapperService mapperService = createMapperService(fieldMapping(b -> {
+            minimalMapping(b);
+            b.field("doc_values", false);
+        }));
+        assertExistsQuery(mapperService);
+        assertParseMinimalWarnings();
+    }
+
     public void testGeoHashValue() throws Exception {
         DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping));
         ParsedDocument doc = mapper.parse(source(b -> b.field("field", stringEncode(1.3, 1.2))));

server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java

@@ -72,6 +72,11 @@ public class GeoShapeFieldMapperTests extends FieldMapperTestCase2<GeoShapeField
         b.field("type", "geo_shape");
     }
 
+    @Override
+    protected void writeFieldValue(XContentBuilder builder) throws IOException {
+        builder.value("POINT (14.0 15.0)");
+    }
+
     public void testDefaultConfiguration() throws IOException {
         DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping));
         Mapper fieldMapper = mapper.mappers().getMapper("field");
@@ -125,7 +130,6 @@ public class GeoShapeFieldMapperTests extends FieldMapperTestCase2<GeoShapeField
         assertFieldWarnings("tree");
     }
 
-
     /**
      * Test that accept_z_value parameter correctly parses
      */
@@ -165,7 +169,6 @@ public class GeoShapeFieldMapperTests extends FieldMapperTestCase2<GeoShapeField
         assertThat(ignoreMalformed.value(), equalTo(false));
     }
 
-
     private void assertFieldWarnings(String... fieldNames) {
         String[] warnings = new String[fieldNames.length];
         for (int i = 0; i < fieldNames.length; ++i) {

server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java

@@ -41,11 +41,25 @@ import static org.hamcrest.Matchers.containsString;
 
 public class IpFieldMapperTests extends MapperTestCase {
 
+    @Override
+    protected void writeFieldValue(XContentBuilder builder) throws IOException {
+        builder.value("::1");
+    }
+
     @Override
     protected void minimalMapping(XContentBuilder b) throws IOException {
         b.field("type", "ip");
     }
 
+    public void testExistsQueryDocValuesDisabled() throws IOException {
+        MapperService mapperService = createMapperService(fieldMapping(b -> {
+            minimalMapping(b);
+            b.field("doc_values", false);
+        }));
+        assertExistsQuery(mapperService);
+        assertParseMinimalWarnings();
+    }
+
     public void testDefaults() throws Exception {
         DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping));
 

server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java

@@ -65,6 +65,7 @@ import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 
 public class KeywordFieldMapperTests extends MapperTestCase {
+
     /**
      * Creates a copy of the lowercase token filter which we use for testing merge errors.
      */
@@ -87,6 +88,33 @@ public class KeywordFieldMapperTests extends MapperTestCase {
 
     }
 
+    @Override
+    protected void writeFieldValue(XContentBuilder builder) throws IOException {
+        builder.value("value");
+    }
+
+    public final void testExistsQueryDocValuesDisabled() throws IOException {
+        MapperService mapperService = createMapperService(fieldMapping(b -> {
+            minimalMapping(b);
+            b.field("doc_values", false);
+            if (randomBoolean()) {
+                b.field("norms", false);
+            }
+        }));
+        assertExistsQuery(mapperService);
+        assertParseMinimalWarnings();
+    }
+
+    public final void testExistsQueryDocValuesDisabledWithNorms() throws IOException {
+        MapperService mapperService = createMapperService(fieldMapping(b -> {
+            minimalMapping(b);
+            b.field("doc_values", false);
+            b.field("norms", true);
+        }));
+        assertExistsQuery(mapperService);
+        assertParseMinimalWarnings();
+    }
+
     @Override
     protected Collection<? extends Plugin> getPlugins() {
         return singletonList(new MockAnalysisPlugin());

server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java

@@ -25,6 +25,7 @@ import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.core.WhitespaceTokenizer;
+import org.apache.lucene.document.FieldType;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.DocValuesFieldExistsQuery;
 import org.apache.lucene.search.FuzzyQuery;
@@ -102,17 +103,21 @@ public class KeywordFieldTypeTests extends FieldTypeTestCase {
     }
 
     public void testExistsQuery() {
-        KeywordFieldType ft = new KeywordFieldType("field");
-        ft.hasNorms = false;
-        assertEquals(new DocValuesFieldExistsQuery("field"), ft.existsQuery(null));
-
-        ft = new KeywordFieldType("field", true, false, Collections.emptyMap());
-        ft.hasNorms = true;
-        assertEquals(new NormsFieldExistsQuery("field"), ft.existsQuery(null));
-
-        ft = new KeywordFieldType("field", true, false, Collections.emptyMap());
-        ft.hasNorms = false;
-        assertEquals(new TermQuery(new Term(FieldNamesFieldMapper.NAME, "field")), ft.existsQuery(null));
+        {
+            KeywordFieldType ft = new KeywordFieldType("field");
+            assertEquals(new DocValuesFieldExistsQuery("field"), ft.existsQuery(null));
+        }
+        {
+            FieldType fieldType = new FieldType();
+            fieldType.setOmitNorms(false);
+            KeywordFieldType ft = new KeywordFieldType("field", false, fieldType, randomBoolean(), null, null, null, 1.0f,
+                Collections.emptyMap());
+            assertEquals(new NormsFieldExistsQuery("field"), ft.existsQuery(null));
+        }
+        {
+            KeywordFieldType ft = new KeywordFieldType("field", true, false, Collections.emptyMap());
+            assertEquals(new TermQuery(new Term(FieldNamesFieldMapper.NAME, "field")), ft.existsQuery(null));
+        }
     }
 
     public void testRangeQuery() {

server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java

@@ -35,7 +35,6 @@ import org.elasticsearch.common.geo.builders.ShapeBuilder;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.geometry.Point;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.plugins.Plugin;
@@ -60,6 +59,11 @@ import static org.mockito.Mockito.when;
 
 public class LegacyGeoShapeFieldMapperTests extends FieldMapperTestCase2<LegacyGeoShapeFieldMapper.Builder> {
 
+    @Override
+    protected void writeFieldValue(XContentBuilder builder) throws IOException {
+        builder.value("POINT (14.0 15.0)");
+    }
+
     @Override
     protected LegacyGeoShapeFieldMapper.Builder newBuilder() {
         return new LegacyGeoShapeFieldMapper.Builder("geoshape");
@@ -183,7 +187,6 @@ public class LegacyGeoShapeFieldMapperTests extends FieldMapperTestCase2<LegacyG
         assertFieldWarnings("tree", "strategy");
     }
 
-
     /**
      * Test that accept_z_value parameter correctly parses
      */
@@ -460,7 +463,7 @@ public class LegacyGeoShapeFieldMapperTests extends FieldMapperTestCase2<LegacyG
             .field("tree", "quadtree")
             .field("strategy", "term").field("precision", "1km")
             .field("tree_levels", 26).field("distance_error_pct", 26)
-            .field("orientation", "cw")))); 
+            .field("orientation", "cw"))));
         assertThat(e.getMessage(), containsString("mapper [field] has different [strategy]"));
         assertThat(e.getMessage(), containsString("mapper [field] has different [tree]"));
         assertThat(e.getMessage(), containsString("mapper [field] has different [tree_levels]"));
@@ -542,15 +545,6 @@ public class LegacyGeoShapeFieldMapperTests extends FieldMapperTestCase2<LegacyG
     }
 
     public void testPointsOnlyDefaultsWithTermStrategy() throws IOException {
-        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1")
-            .startObject("properties").startObject("location")
-            .field("type", "geo_shape")
-            .field("tree", "quadtree")
-            .field("precision", "10m")
-            .field("strategy", "term")
-            .endObject().endObject()
-            .endObject().endObject());
-
         DocumentMapper mapper = createDocumentMapper(
             fieldMapping(b -> b.field("type", "geo_shape").field("tree", "quadtree").field("precision", "10m").field("strategy", "term"))
         );
@@ -569,7 +563,6 @@ public class LegacyGeoShapeFieldMapperTests extends FieldMapperTestCase2<LegacyG
         assertFieldWarnings("tree", "precision", "strategy");
     }
 
-
     public void testPointsOnlyFalseWithTermStrategy() throws Exception {
         Exception e = expectThrows(
             MapperParsingException.class,

server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java

@@ -58,6 +58,20 @@ public class NumberFieldMapperTests extends AbstractNumericFieldMapperTestCase {
         b.field("type", "long");
     }
 
+    @Override
+    protected void writeFieldValue(XContentBuilder builder) throws IOException {
+        builder.value(123);
+    }
+
+    public void testExistsQueryDocValuesDisabled() throws IOException {
+        MapperService mapperService = createMapperService(fieldMapping(b -> {
+            minimalMapping(b);
+            b.field("doc_values", false);
+        }));
+        assertExistsQuery(mapperService);
+        assertParseMinimalWarnings();
+    }
+
     @Override
     public void doTestDefaults(String type) throws Exception {
         XContentBuilder mapping = fieldMapping(b -> b.field("type", type));

server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java

@@ -73,6 +73,20 @@ public class RangeFieldMapperTests extends AbstractNumericFieldMapperTestCase {
         b.field("type", "long_range");
     }
 
+    @Override
+    protected void writeFieldValue(XContentBuilder builder) throws IOException {
+        builder.startObject().field(getFromField(), getFrom("long_range")).field(getToField(), getTo("long_range")).endObject();
+    }
+
+    public void testExistsQueryDocValuesDisabled() throws IOException {
+        MapperService mapperService = createMapperService(fieldMapping(b -> {
+            minimalMapping(b);
+            b.field("doc_values", false);
+        }));
+        assertExistsQuery(mapperService);
+        assertParseMinimalWarnings();
+    }
+
     @Override
     protected void assertParseMaximalWarnings() {
         assertWarnings("Parameter [boost] on field [field] is deprecated and will be removed in 8.0");

server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java

@@ -91,6 +91,41 @@ import static org.hamcrest.core.Is.is;
 
 public class TextFieldMapperTests extends FieldMapperTestCase2<TextFieldMapper.Builder> {
 
+    @Override
+    protected void writeFieldValue(XContentBuilder builder) throws IOException {
+        builder.value(1234);
+    }
+
+    public final void testExistsQueryIndexDisabled() throws IOException {
+        MapperService mapperService = createMapperService(fieldMapping(b -> {
+            minimalMapping(b);
+            b.field("index", false);
+            b.field("norms", false);
+        }));
+        assertExistsQuery(mapperService);
+        assertParseMinimalWarnings();
+    }
+
+    public final void testExistsQueryIndexDisabledStoreTrue() throws IOException {
+        MapperService mapperService = createMapperService(fieldMapping(b -> {
+            minimalMapping(b);
+            b.field("index", false);
+            b.field("norms", false);
+            b.field("store", true);
+        }));
+        assertExistsQuery(mapperService);
+        assertParseMinimalWarnings();
+    }
+
+    public final void testExistsQueryWithNorms() throws IOException {
+        MapperService mapperService = createMapperService(fieldMapping(b -> {
+            minimalMapping(b);
+            b.field("norms", false);
+        }));
+        assertExistsQuery(mapperService);
+        assertParseMinimalWarnings();
+    }
+
     @Override
     protected TextFieldMapper.Builder newBuilder() {
         return new TextFieldMapper.Builder("text")

test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java

@@ -49,6 +49,7 @@ import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.ScriptPlugin;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -232,6 +233,10 @@ public abstract class MapperServiceTestCase extends ESTestCase {
         when(queryShardContext.simpleMatchToIndexNames(anyObject())).thenAnswer(
             inv -> mapperService.simpleMatchToFullName(inv.getArguments()[0].toString())
         );
+        when(queryShardContext.allowExpensiveQueries()).thenReturn(true);
+        when(queryShardContext.lookup()).thenReturn(new SearchLookup(mapperService, (ft, s) -> {
+            throw new UnsupportedOperationException("search lookup not available");
+        }));
         return queryShardContext;
     }
 }

test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java

@@ -19,8 +19,16 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.IndexableFieldType;
 import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.DocValuesFieldExistsQuery;
 import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.NormsFieldExistsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -30,6 +38,7 @@ import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
+import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.lookup.SearchLookup;
@@ -44,6 +53,7 @@ import java.util.function.Supplier;
 
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.instanceOf;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -53,6 +63,104 @@ import static org.mockito.Mockito.when;
 public abstract class MapperTestCase extends MapperServiceTestCase {
     protected abstract void minimalMapping(XContentBuilder b) throws IOException;
 
+    /**
+     * Writes the field and a sample value for it to the provided {@link XContentBuilder}.
+     * To be overridden in case the field should not be written at all in documents,
+     * like in the case of runtime fields.
+     */
+    protected void writeField(XContentBuilder builder) throws IOException {
+        builder.field("field");
+        writeFieldValue(builder);
+    }
+
+    /**
+     * Writes a sample value for the field to the provided {@link XContentBuilder}.
+     */
+    protected abstract void writeFieldValue(XContentBuilder builder) throws IOException;
+
+    /**
+     * This test verifies that the exists query created is the appropriate one, and aligns with the data structures
+     * being created for a document with a value for the field. This can only be verified for the minimal mapping.
+     * Field types that allow configurable doc_values or norms should write their own tests that creates the different
+     * mappings combinations and invoke {@link #assertExistsQuery(MapperService)} to verify the behaviour.
+     */
+    public final void testExistsQueryMinimalMapping() throws IOException {
+        MapperService mapperService = createMapperService(fieldMapping(this::minimalMapping));
+        assertExistsQuery(mapperService);
+        assertParseMinimalWarnings();
+    }
+
+    protected void assertExistsQuery(MapperService mapperService) throws IOException {
+        ParseContext.Document fields = mapperService.documentMapper().parse(source(this::writeField)).rootDoc();
+        QueryShardContext queryShardContext = createQueryShardContext(mapperService);
+        MappedFieldType fieldType = mapperService.fieldType("field");
+        Query query = fieldType.existsQuery(queryShardContext);
+        assertExistsQuery(fieldType, query, fields);
+    }
+
+    protected void assertExistsQuery(MappedFieldType fieldType, Query query, ParseContext.Document fields) {
+        if (fieldType.hasDocValues()) {
+            assertThat(query, instanceOf(DocValuesFieldExistsQuery.class));
+            DocValuesFieldExistsQuery fieldExistsQuery = (DocValuesFieldExistsQuery)query;
+            assertEquals("field", fieldExistsQuery.getField());
+            assertDocValuesField(fields, "field");
+            assertNoFieldNamesField(fields);
+        } else if (fieldType.getTextSearchInfo().hasNorms()) {
+            assertThat(query, instanceOf(NormsFieldExistsQuery.class));
+            NormsFieldExistsQuery normsFieldExistsQuery = (NormsFieldExistsQuery) query;
+            assertEquals("field", normsFieldExistsQuery.getField());
+            assertHasNorms(fields, "field");
+            assertNoDocValuesField(fields, "field");
+            assertNoFieldNamesField(fields);
+        } else {
+            assertThat(query, instanceOf(TermQuery.class));
+            TermQuery termQuery = (TermQuery) query;
+            assertEquals(FieldNamesFieldMapper.NAME, termQuery.getTerm().field());
+            //we always perform a term query against _field_names, even when the field
+            // is not added to _field_names because it is not indexed nor stored
+            assertEquals("field", termQuery.getTerm().text());
+            assertNoDocValuesField(fields, "field");
+            if (fieldType.isSearchable() || fieldType.isStored()) {
+                assertNotNull(fields.getField(FieldNamesFieldMapper.NAME));
+            } else {
+                assertNoFieldNamesField(fields);
+            }
+        }
+    }
+
+    protected static void assertNoFieldNamesField(ParseContext.Document fields) {
+        assertNull(fields.getField(FieldNamesFieldMapper.NAME));
+    }
+
+    protected static void assertHasNorms(ParseContext.Document doc, String field) {
+        IndexableField[] fields = doc.getFields(field);
+        for (IndexableField indexableField : fields) {
+            IndexableFieldType indexableFieldType = indexableField.fieldType();
+            if (indexableFieldType.indexOptions() != IndexOptions.NONE) {
+                assertFalse(indexableFieldType.omitNorms());
+                return;
+            }
+        }
+        fail("field [" + field + "] should be indexed but it isn't");
+    }
+
+    protected static void assertDocValuesField(ParseContext.Document doc, String field) {
+        IndexableField[] fields = doc.getFields(field);
+        for (IndexableField indexableField : fields) {
+            if (indexableField.fieldType().docValuesType().equals(DocValuesType.NONE) == false) {
+                return;
+            }
+        }
+        fail("doc_values not present for field [" + field + "]");
+    }
+
+    protected static void assertNoDocValuesField(ParseContext.Document doc, String field) {
+        IndexableField[] fields = doc.getFields(field);
+        for (IndexableField indexableField : fields) {
+            assertEquals(DocValuesType.NONE, indexableField.fieldType().docValuesType());
+        }
+    }
+
     public final void testEmptyName() {
         MapperParsingException e = expectThrows(MapperParsingException.class, () -> createMapperService(mapping(b -> {
             b.startObject("");

test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java

@@ -20,14 +20,9 @@
 package org.elasticsearch.index.mapper;
 
 import org.apache.lucene.document.FieldType;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.DocValuesFieldExistsQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.io.IOException;
@@ -70,15 +65,6 @@ public class MockFieldMapper extends FieldMapper {
         public String typeName() {
             return "faketype";
         }
-
-        @Override
-        public Query existsQuery(QueryShardContext context) {
-            if (hasDocValues()) {
-                return new DocValuesFieldExistsQuery(name());
-            } else {
-                return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name()));
-            }
-        }
     }
 
     @Override

x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java

@@ -15,7 +15,6 @@ import org.apache.lucene.index.BinaryDocValues;
 import org.apache.lucene.index.DocValues;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.search.DocValuesFieldExistsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.store.ByteArrayDataInput;
@@ -280,16 +279,6 @@ public class HistogramFieldMapper extends FieldMapper {
             };
         }
 
-        @Override
-        public Query existsQuery(QueryShardContext context) {
-            if (hasDocValues()) {
-                return new DocValuesFieldExistsQuery(name());
-            } else {
-                throw new QueryShardException(context, "field  " + name() + " of type [" + CONTENT_TYPE + "] " +
-                    "has no doc values and cannot be searched");
-            }
-        }
-
         @Override
         public Query termQuery(Object value, QueryShardContext context) {
             throw new QueryShardException(context, "[" + CONTENT_TYPE + "] field do not support searching, " +
@@ -374,27 +363,25 @@ public class HistogramFieldMapper extends FieldMapper {
                     + name() + "], expected same length from [" + VALUES_FIELD.getPreferredName() +"] and " +
                     "[" + COUNTS_FIELD.getPreferredName() +"] but got [" + values.size() + " != " + counts.size() +"]");
             }
-            if (fieldType().hasDocValues()) {
-                ByteBuffersDataOutput dataOutput = new ByteBuffersDataOutput();
-                for (int i = 0; i < values.size(); i++) {
-                    int count = counts.get(i);
-                    if (count < 0) {
-                        throw new MapperParsingException("error parsing field ["
-                            + name() + "], ["+ COUNTS_FIELD + "] elements must be >= 0 but got " + counts.get(i));
-                    } else if (count > 0) {
-                        // we do not add elements with count == 0
-                        dataOutput.writeVInt(count);
-                        dataOutput.writeLong(Double.doubleToRawLongBits(values.get(i)));
-                    }
-                }
-                BytesRef docValue = new BytesRef(dataOutput.toArrayCopy(), 0, Math.toIntExact(dataOutput.size()));
-                Field field = new BinaryDocValuesField(name(), docValue);
-                if (context.doc().getByKey(fieldType().name()) != null) {
-                    throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() +
-                        "] doesn't not support indexing multiple values for the same field in the same document");
+            ByteBuffersDataOutput dataOutput = new ByteBuffersDataOutput();
+            for (int i = 0; i < values.size(); i++) {
+                int count = counts.get(i);
+                if (count < 0) {
+                    throw new MapperParsingException("error parsing field ["
+                        + name() + "], ["+ COUNTS_FIELD + "] elements must be >= 0 but got " + counts.get(i));
+                } else if (count > 0) {
+                    // we do not add elements with count == 0
+                    dataOutput.writeVInt(count);
+                    dataOutput.writeLong(Double.doubleToRawLongBits(values.get(i)));
                 }
-                context.doc().addWithKey(fieldType().name(), field);
             }
+            BytesRef docValue = new BytesRef(dataOutput.toArrayCopy(), 0, Math.toIntExact(dataOutput.size()));
+            Field field = new BinaryDocValuesField(name(), docValue);
+            if (context.doc().getByKey(fieldType().name()) != null) {
+                throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() +
+                    "] doesn't not support indexing multiple values for the same field in the same document");
+            }
+            context.doc().addWithKey(fieldType().name(), field);
 
         } catch (Exception ex) {
             if (ignoreMalformed.value() == false) {

x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java

@@ -26,6 +26,11 @@ import static org.hamcrest.Matchers.nullValue;
 
 public class HistogramFieldMapperTests extends FieldMapperTestCase2<HistogramFieldMapper.Builder> {
 
+    @Override
+    protected void writeFieldValue(XContentBuilder builder) throws IOException {
+        builder.startObject().field("values", new double[] { 2, 3 }).field("counts", new int[] { 0, 4 }).endObject();
+    }
+
     @Override
     protected Set<String> unsupportedProperties() {
         return Set.of("analyzer", "similarity", "doc_values", "store", "index");

x-pack/plugin/mapper-constant-keyword/src/internalClusterTest/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java

@@ -6,15 +6,19 @@
 
 package org.elasticsearch.xpack.constantkeyword.mapper;
 
+import org.apache.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.search.Query;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
 import org.elasticsearch.index.mapper.MapperTestCase;
+import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.ValueFetcher;
 import org.elasticsearch.plugins.Plugin;
@@ -26,8 +30,26 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 
+import static org.hamcrest.Matchers.instanceOf;
+
 public class ConstantKeywordFieldMapperTests extends MapperTestCase {
 
+    @Override
+    protected void writeField(XContentBuilder builder) {
+        //do nothing
+    }
+
+    @Override
+    protected void writeFieldValue(XContentBuilder builder) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    protected void assertExistsQuery(MappedFieldType fieldType, Query query, ParseContext.Document fields) {
+        assertThat(query, instanceOf(MatchNoDocsQuery.class));
+        assertNoFieldNamesField(fields);
+    }
+
     @Override
     protected Collection<Plugin> getPlugins() {
         return List.of(new ConstantKeywordMapperPlugin());

x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java

@@ -13,12 +13,10 @@ import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.OrdinalMap;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BoostQuery;
-import org.apache.lucene.search.DocValuesFieldExistsQuery;
 import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.SortField;
-import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.search.AutomatonQueries;
@@ -37,7 +35,6 @@ import org.elasticsearch.index.fielddata.plain.AbstractLeafOrdinalsFieldData;
 import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData;
 import org.elasticsearch.index.mapper.DynamicKeyFieldMapper;
 import org.elasticsearch.index.mapper.FieldMapper;
-import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
@@ -463,15 +460,6 @@ public final class FlatObjectFieldMapper extends DynamicKeyFieldMapper {
             return binaryValue.utf8ToString();
         }
 
-        @Override
-        public Query existsQuery(QueryShardContext context) {
-            if (hasDocValues()) {
-                return new DocValuesFieldExistsQuery(name());
-            } else {
-                return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name()));
-            }
-        }
-
         @Override
         public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier<SearchLookup> searchLookup) {
             failIfNoDocValues();

x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptMappedFieldType.java

@@ -24,7 +24,6 @@ import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.io.IOException;
 import java.time.ZoneId;
-import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 
@@ -89,9 +88,6 @@ abstract class AbstractScriptMappedFieldType<LeafFactory> extends MappedFieldTyp
         return leafFactory(context.lookup().forkAndTrackFieldReferences(name()));
     }
 
-    @Override
-    public abstract Query termsQuery(List<?> values, QueryShardContext context);
-
     @Override
     public final Query rangeQuery(
         Object lowerTerm,
@@ -154,9 +150,6 @@ abstract class AbstractScriptMappedFieldType<LeafFactory> extends MappedFieldTyp
         throw new IllegalArgumentException(unsupported("regexp", "keyword and text"));
     }
 
-    @Override
-    public abstract Query existsQuery(QueryShardContext context);
-
     @Override
     public Query phraseQuery(TokenStream stream, int slop, boolean enablePositionIncrements) throws IOException {
         throw new IllegalArgumentException(unsupported("phrase", "text"));

x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldMapperTests.java

@@ -6,6 +6,7 @@
 
 package org.elasticsearch.xpack.runtimefields.mapper;
 
+import org.apache.lucene.search.Query;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.CheckedConsumer;
@@ -22,6 +23,7 @@ import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.MapperTestCase;
+import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
 import org.elasticsearch.plugins.Plugin;
@@ -30,6 +32,7 @@ import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptEngine;
 import org.elasticsearch.xpack.runtimefields.RuntimeFields;
+import org.elasticsearch.xpack.runtimefields.query.StringScriptFieldExistsQuery;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -51,6 +54,22 @@ public class RuntimeFieldMapperTests extends MapperTestCase {
         Arrays.sort(runtimeTypes);
     }
 
+    @Override
+    protected void writeField(XContentBuilder builder) {
+        // do nothing
+    }
+
+    @Override
+    protected void writeFieldValue(XContentBuilder builder) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    protected void assertExistsQuery(MappedFieldType fieldType, Query query, ParseContext.Document fields) {
+        assertThat(query, instanceOf(StringScriptFieldExistsQuery.class));
+        assertNoFieldNamesField(fields);
+    }
+
     @Override
     protected void minimalMapping(XContentBuilder b) throws IOException {
         b.field("type", "runtime").field("runtime_type", "keyword");

x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java

@@ -127,7 +127,7 @@ public class PointFieldMapper extends AbstractPointGeometryFieldMapper<List<Pars
     }
 
     public static class PointFieldType extends AbstractPointGeometryFieldType<List<ParsedCartesianPoint>, List<? extends CartesianPoint>> {
-        public PointFieldType(String name, boolean indexed, boolean stored, boolean hasDocValues, Map<String, String> meta) {
+        private PointFieldType(String name, boolean indexed, boolean stored, boolean hasDocValues, Map<String, String> meta) {
             super(name, indexed, stored, hasDocValues, meta);
         }
 

x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java

@@ -10,7 +10,6 @@ package org.elasticsearch.xpack.vectors.mapper;
 import org.apache.lucene.document.BinaryDocValuesField;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.search.DocValuesFieldExistsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.Version;
@@ -124,11 +123,6 @@ public class DenseVectorFieldMapper extends FieldMapper {
                 "Field [" + name() + "] of type [" + typeName() + "] doesn't support docvalue_fields or aggregations");
         }
 
-        @Override
-        public Query existsQuery(QueryShardContext context) {
-            return new DocValuesFieldExistsQuery(name());
-        }
-
         @Override
         public boolean isAggregatable() {
             return false;

x-pack/plugin/versionfield/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java

@@ -122,7 +122,7 @@ public class VersionStringFieldMapper extends ParametrizedFieldMapper {
 
     public static final class VersionStringFieldType extends TermBasedFieldType {
 
-        public VersionStringFieldType(String name, FieldType fieldType, Map<String, String> meta) {
+        private VersionStringFieldType(String name, FieldType fieldType, Map<String, String> meta) {
             super(name, true, false, true, new TextSearchInfo(fieldType, null, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER), meta);
             setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
         }

x-pack/plugin/versionfield/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java

@@ -46,6 +46,11 @@ public class VersionStringFieldMapperTests extends MapperTestCase {
         b.field("type", "version");
     }
 
+    @Override
+    protected void writeFieldValue(XContentBuilder builder) throws IOException {
+        builder.value("1.2.3");
+    }
+
     public void testDefaults() throws Exception {
         XContentBuilder mapping = fieldMapping(this::minimalMapping);
         DocumentMapper mapper = createDocumentMapper(mapping);

x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java

@@ -212,7 +212,7 @@ public class WildcardFieldMapper extends FieldMapper {
 
         static Analyzer lowercaseNormalizer = new LowercaseNormalizer();
 
-        public WildcardFieldType(String name, FieldType fieldType, Map<String, String> meta) {
+        private WildcardFieldType(String name, FieldType fieldType, Map<String, String> meta) {
             super(name, true, fieldType.stored(), true,
                 new TextSearchInfo(fieldType, null, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER), meta);
             setIndexAnalyzer(WILDCARD_ANALYZER);
@@ -841,12 +841,6 @@ public class WildcardFieldMapper extends FieldMapper {
             return KeywordFieldMapper.CONTENT_TYPE;
         }
 
-
-        @Override
-        public Query existsQuery(QueryShardContext context) {
-            return new DocValuesFieldExistsQuery(name());
-        }
-
         @Override
         public Query termQuery(Object value, QueryShardContext context) {
             String searchTerm = BytesRefs.toString(value);