Remove payload option from completion suggester

The payload option was introduced with the new completion
suggester implementation in v5, as a stop gap solution
to return additional metadata with suggestions.

Now we can return associated documents with suggestions
(#19536) through fetch phase using stored field (_source).
The additional fetch phase ensures that we only fetch
the _source for the global top-N suggestions instead of
fetching _source of top results for each shard.

core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java

@@ -78,7 +78,6 @@ public class CompletionSuggester extends Suggester<CompletionSuggestionContext>
             TopSuggestDocsCollector collector = new TopDocumentsCollector(suggestionContext.getSize());
             suggest(searcher, suggestionContext.toQuery(), collector);
             int numResult = 0;
-            List<LeafReaderContext> leaves = searcher.getIndexReader().leaves();
             for (TopSuggestDocs.SuggestScoreDoc suggestScoreDoc : collector.get().scoreLookupDocs()) {
                 TopDocumentsCollector.SuggestDoc suggestDoc = (TopDocumentsCollector.SuggestDoc) suggestScoreDoc;
                 // collect contexts
@@ -86,31 +85,9 @@ public class CompletionSuggester extends Suggester<CompletionSuggestionContext>
                 if (fieldType.hasContextMappings() && suggestDoc.getContexts().isEmpty() == false) {
                     contexts = fieldType.getContextMappings().getNamedContexts(suggestDoc.getContexts());
                 }
-                // collect payloads
-                final Map<String, List<Object>> payload = new HashMap<>(0);
-                List<String> payloadFields = suggestionContext.getPayloadFields();
-                if (payloadFields.isEmpty() == false) {
-                    final int readerIndex = ReaderUtil.subIndex(suggestDoc.doc, leaves);
-                    final LeafReaderContext subReaderContext = leaves.get(readerIndex);
-                    final int subDocId = suggestDoc.doc - subReaderContext.docBase;
-                    for (String field : payloadFields) {
-                        MapperService mapperService = suggestionContext.getShardContext().getMapperService();
-                        MappedFieldType payloadFieldType = mapperService.fullName(field);
-                        if (payloadFieldType != null) {
-                            QueryShardContext shardContext = suggestionContext.getShardContext();
-                            final AtomicFieldData data = shardContext.getForField(payloadFieldType)
-                                .load(subReaderContext);
-                            final ScriptDocValues scriptValues = data.getScriptValues();
-                            scriptValues.setNextDocId(subDocId);
-                            payload.put(field, new ArrayList<>(scriptValues.getValues()));
-                        } else {
-                            throw new IllegalArgumentException("payload field [" + field + "] does not exist");
-                        }
-                    }
-                }
                 if (numResult++ < suggestionContext.getSize()) {
                     CompletionSuggestion.Entry.Option option = new CompletionSuggestion.Entry.Option(suggestDoc.doc,
-                        new Text(suggestDoc.key.toString()), suggestDoc.score, contexts, payload);
+                        new Text(suggestDoc.key.toString()), suggestDoc.score, contexts);
                     completionSuggestEntry.addOption(option);
                 } else {
                     break;

core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestion.java

@@ -194,14 +194,12 @@ public final class CompletionSuggestion extends Suggest.Suggestion<CompletionSug
 
         public static class Option extends Suggest.Suggestion.Entry.Option {
             private Map<String, Set<CharSequence>> contexts;
-            private Map<String, List<Object>> payload;
             private ScoreDoc doc;
             private InternalSearchHit hit;
 
-            public Option(int docID, Text text, float score, Map<String, Set<CharSequence>> contexts, Map<String, List<Object>> payload) {
+            public Option(int docID, Text text, float score, Map<String, Set<CharSequence>> contexts) {
                 super(text, score);
                 this.doc = new ScoreDoc(docID, score);
-                this.payload = payload;
                 this.contexts = contexts;
             }
 
@@ -216,10 +214,6 @@ public final class CompletionSuggestion extends Suggest.Suggestion<CompletionSug
                 throw new UnsupportedOperationException();
             }
 
-            public Map<String, List<Object>> getPayload() {
-                return payload;
-            }
-
             public Map<String, Set<CharSequence>> getContexts() {
                 return contexts;
             }
@@ -248,17 +242,6 @@ public final class CompletionSuggestion extends Suggest.Suggestion<CompletionSug
                 } else {
                     builder.field("score", getScore());
                 }
-                if (payload.size() > 0) {
-                    builder.startObject("payload");
-                    for (Map.Entry<String, List<Object>> entry : payload.entrySet()) {
-                        builder.startArray(entry.getKey());
-                        for (Object payload : entry.getValue()) {
-                            builder.value(payload);
-                        }
-                        builder.endArray();
-                    }
-                    builder.endObject();
-                }
                 if (contexts.size() > 0) {
                     builder.startObject("contexts");
                     for (Map.Entry<String, Set<CharSequence>> entry : contexts.entrySet()) {
@@ -281,17 +264,6 @@ public final class CompletionSuggestion extends Suggest.Suggestion<CompletionSug
                     this.hit = InternalSearchHit.readSearchHit(in,
                         InternalSearchHits.streamContext().streamShardTarget(ShardTargetType.STREAM));
                 }
-                int payloadSize = in.readInt();
-                this.payload = new LinkedHashMap<>(payloadSize);
-                for (int i = 0; i < payloadSize; i++) {
-                    String payloadName = in.readString();
-                    int nValues = in.readVInt();
-                    List<Object> values = new ArrayList<>(nValues);
-                    for (int j = 0; j < nValues; j++) {
-                        values.add(in.readGenericValue());
-                    }
-                    this.payload.put(payloadName, values);
-                }
                 int contextSize = in.readInt();
                 this.contexts = new LinkedHashMap<>(contextSize);
                 for (int i = 0; i < contextSize; i++) {
@@ -315,15 +287,6 @@ public final class CompletionSuggestion extends Suggest.Suggestion<CompletionSug
                 } else {
                     out.writeBoolean(false);
                 }
-                out.writeInt(payload.size());
-                for (Map.Entry<String, List<Object>> entry : payload.entrySet()) {
-                    out.writeString(entry.getKey());
-                    List<Object> values = entry.getValue();
-                    out.writeVInt(values.size());
-                    for (Object value : values) {
-                        out.writeGenericValue(value);
-                    }
-                }
                 out.writeInt(contexts.size());
                 for (Map.Entry<String, Set<CharSequence>> entry : contexts.entrySet()) {
                     out.writeString(entry.getKey());
@@ -341,14 +304,6 @@ public final class CompletionSuggestion extends Suggest.Suggestion<CompletionSug
                 stringBuilder.append(getText());
                 stringBuilder.append(" score:");
                 stringBuilder.append(getScore());
-                stringBuilder.append(" payload:[");
-                for (Map.Entry<String, List<Object>> entry : payload.entrySet()) {
-                    stringBuilder.append(" ");
-                    stringBuilder.append(entry.getKey());
-                    stringBuilder.append(":");
-                    stringBuilder.append(entry.getValue());
-                }
-                stringBuilder.append("]");
                 stringBuilder.append(" context:[");
                 for (Map.Entry<String, Set<CharSequence>> entry: contexts.entrySet()) {
                     stringBuilder.append(" ");

core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java

@@ -62,7 +62,6 @@ import java.util.Objects;
  */
 public class CompletionSuggestionBuilder extends SuggestionBuilder<CompletionSuggestionBuilder> {
     static final String SUGGESTION_NAME = "completion";
-    static final ParseField PAYLOAD_FIELD = new ParseField("payload");
     static final ParseField CONTEXTS_FIELD = new ParseField("contexts", "context");
 
     /**
@@ -78,7 +77,6 @@ public class CompletionSuggestionBuilder extends SuggestionBuilder<CompletionSug
     private static ObjectParser<CompletionSuggestionBuilder.InnerBuilder, ParseFieldMatcherSupplier> TLP_PARSER =
         new ObjectParser<>(SUGGESTION_NAME, null);
     static {
-        TLP_PARSER.declareStringArray(CompletionSuggestionBuilder.InnerBuilder::payload, PAYLOAD_FIELD);
         TLP_PARSER.declareField((parser, completionSuggestionContext, context) -> {
                 if (parser.currentToken() == XContentParser.Token.VALUE_BOOLEAN) {
                     if (parser.booleanValue()) {
@@ -108,7 +106,6 @@ public class CompletionSuggestionBuilder extends SuggestionBuilder<CompletionSug
     protected FuzzyOptions fuzzyOptions;
     protected RegexOptions regexOptions;
     protected BytesReference contextBytes = null;
-    protected List<String> payloadFields = Collections.emptyList();
 
     public CompletionSuggestionBuilder(String field) {
         super(field);
@@ -123,7 +120,6 @@ public class CompletionSuggestionBuilder extends SuggestionBuilder<CompletionSug
         fuzzyOptions = in.fuzzyOptions;
         regexOptions = in.regexOptions;
         contextBytes = in.contextBytes;
-        payloadFields = in.payloadFields;
     }
 
     /**
@@ -131,8 +127,6 @@ public class CompletionSuggestionBuilder extends SuggestionBuilder<CompletionSug
      */
     public CompletionSuggestionBuilder(StreamInput in) throws IOException {
         super(in);
-        payloadFields = new ArrayList<>();
-        Collections.addAll(payloadFields, in.readStringArray());
         fuzzyOptions = in.readOptionalWriteable(FuzzyOptions::new);
         regexOptions = in.readOptionalWriteable(RegexOptions::new);
         contextBytes = in.readOptionalBytesReference();
@@ -140,7 +134,6 @@ public class CompletionSuggestionBuilder extends SuggestionBuilder<CompletionSug
 
     @Override
     public void doWriteTo(StreamOutput out) throws IOException {
-        out.writeStringArray(payloadFields.toArray(new String[payloadFields.size()]));
         out.writeOptionalWriteable(fuzzyOptions);
         out.writeOptionalWriteable(regexOptions);
         out.writeOptionalBytesReference(contextBytes);
@@ -194,16 +187,6 @@ public class CompletionSuggestionBuilder extends SuggestionBuilder<CompletionSug
         return this;
     }
 
-    /**
-     * Sets the fields to be returned as suggestion payload.
-     * Note: Only doc values enabled fields are supported
-     */
-    public CompletionSuggestionBuilder payload(List<String> fields) {
-        Objects.requireNonNull(fields, "payload must not be null");
-        this.payloadFields = fields;
-        return this;
-    }
-
     /**
      * Sets query contexts for completion
      * @param queryContexts named query contexts
@@ -348,13 +331,6 @@ public class CompletionSuggestionBuilder extends SuggestionBuilder<CompletionSug
 
     @Override
     protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
-        if (payloadFields.isEmpty() == false) {
-            builder.startArray(PAYLOAD_FIELD.getPreferredName());
-            for (String field : payloadFields) {
-                builder.value(field);
-            }
-            builder.endArray();
-        }
         if (fuzzyOptions != null) {
             fuzzyOptions.toXContent(builder, params);
         }
@@ -388,7 +364,6 @@ public class CompletionSuggestionBuilder extends SuggestionBuilder<CompletionSug
         // copy over common settings to each suggestion builder
         final MapperService mapperService = context.getMapperService();
         populateCommonFields(mapperService, suggestionContext);
-        suggestionContext.setPayloadFields(payloadFields);
         suggestionContext.setFuzzyOptions(fuzzyOptions);
         suggestionContext.setRegexOptions(regexOptions);
         MappedFieldType mappedFieldType = mapperService.fullName(suggestionContext.getField());
@@ -449,14 +424,13 @@ public class CompletionSuggestionBuilder extends SuggestionBuilder<CompletionSug
 
     @Override
     protected boolean doEquals(CompletionSuggestionBuilder other) {
-        return Objects.equals(payloadFields, other.payloadFields) &&
-            Objects.equals(fuzzyOptions, other.fuzzyOptions) &&
+        return Objects.equals(fuzzyOptions, other.fuzzyOptions) &&
             Objects.equals(regexOptions, other.regexOptions) &&
             Objects.equals(contextBytes, other.contextBytes);
     }
 
     @Override
     protected int doHashCode() {
-        return Objects.hash(payloadFields, fuzzyOptions, regexOptions, contextBytes);
+        return Objects.hash(fuzzyOptions, regexOptions, contextBytes);
     }
 }

core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionContext.java

@@ -45,7 +45,6 @@ public class CompletionSuggestionContext extends SuggestionSearchContext.Suggest
     private FuzzyOptions fuzzyOptions;
     private RegexOptions regexOptions;
     private Map<String, List<ContextMapping.InternalQueryContext>> queryContexts = Collections.emptyMap();
-    private List<String> payloadFields = Collections.emptyList();
     private CompletionFieldMapper2x.CompletionFieldType fieldType2x;
     private List<ContextQuery> contextQueries;
 
@@ -73,14 +72,6 @@ public class CompletionSuggestionContext extends SuggestionSearchContext.Suggest
         this.queryContexts = queryContexts;
     }
 
-    void setPayloadFields(List<String> fields) {
-        this.payloadFields = fields;
-    }
-
-    List<String> getPayloadFields() {
-        return payloadFields;
-    }
-
     public FuzzyOptions getFuzzyOptions() {
         return fuzzyOptions;
     }

core/src/test/java/org/elasticsearch/search/controller/SearchPhaseControllerTests.java

@@ -149,7 +149,7 @@ public class SearchPhaseControllerTests extends ESTestCase {
                 float maxScore = randomIntBetween(suggestion.getSize(), (int) Float.MAX_VALUE);
                 for (int i = 0; i < optionSize; i++) {
                     completionEntry.addOption(new CompletionSuggestion.Entry.Option(i, new Text(""), maxScore,
-                        Collections.emptyMap(), Collections.emptyMap()));
+                        Collections.emptyMap()));
                     float dec = randomIntBetween(0, optionSize);
                     if (dec <= maxScore) {
                         maxScore -= dec;

core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java

@@ -246,156 +246,6 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase {
         assertSuggestions("foo", fuzzyPrefix, outputs);
     }
 
-    public void testSuggestWithNumericPayload() throws Exception {
-        final CompletionMappingBuilder mapping = new CompletionMappingBuilder();
-        createIndexAndMapping(mapping);
-        int numDocs = 10;
-        List<IndexRequestBuilder> indexRequestBuilders = new ArrayList<>();
-        for (int i = 0; i < numDocs; i++) {
-            XContentBuilder source= jsonBuilder()
-                    .startObject()
-                    .field(FIELD, "suggestion" + i)
-                    .field("count", i)
-                    .endObject();
-            indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i).setSource(source));
-        }
-        indexRandom(true, indexRequestBuilders);
-
-        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg").
-            size(numDocs).payload(Collections.singletonList("count"));
-        SearchResponse searchResponse = client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", prefix))
-            .execute().actionGet();
-        assertNoFailures(searchResponse);
-        CompletionSuggestion completionSuggestion = searchResponse.getSuggest().getSuggestion("foo");
-        CompletionSuggestion.Entry options = completionSuggestion.getEntries().get(0);
-        assertThat(options.getOptions().size(), equalTo(numDocs));
-        for (CompletionSuggestion.Entry.Option option : options) {
-            Map<String, List<Object>> payloads = option.getPayload();
-            assertThat(payloads.keySet(), contains("count"));
-        }
-    }
-
-    public void testMissingPayloadField() throws Exception {
-        final CompletionMappingBuilder mapping = new CompletionMappingBuilder();
-        createIndexAndMapping(mapping);
-        List<IndexRequestBuilder> indexRequestBuilders = Arrays.asList(
-                client().prepareIndex(INDEX, TYPE, "1").setSource(FIELD, "suggestion", "test_field", "test"),
-                client().prepareIndex(INDEX, TYPE, "2").setSource(FIELD, "suggestion")
-        );
-        indexRandom(true, indexRequestBuilders);
-        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg")
-            .payload(Collections.singletonList("test_field"));
-        SearchResponse searchResponse = client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", prefix))
-            .execute().actionGet();
-        assertNoFailures(searchResponse);
-        CompletionSuggestion completionSuggestion = searchResponse.getSuggest().getSuggestion("foo");
-        CompletionSuggestion.Entry options = completionSuggestion.getEntries().get(0);
-        assertThat(options.getOptions().size(), equalTo(2));
-        for (CompletionSuggestion.Entry.Option option : options.getOptions()) {
-            assertThat(option.getPayload().keySet(), contains("test_field"));
-        }
-    }
-
-    public void testPayload() throws Exception {
-        final CompletionMappingBuilder mapping = new CompletionMappingBuilder();
-        createIndexAndMapping(mapping);
-        List<IndexRequestBuilder> indexRequestBuilders = new ArrayList<>();
-        XContentBuilder source = jsonBuilder()
-                .startObject()
-                .startObject(FIELD)
-                .field("input", "suggest")
-                .field("weight", 1)
-                .endObject()
-                .field("title", "title1")
-                .field("count", 1)
-                .endObject();
-        indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "1").setSource(source));
-        source = jsonBuilder()
-                .startObject()
-                .startObject(FIELD)
-                .field("input", "suggestion")
-                .field("weight", 2)
-                .endObject()
-                .field("title", "title2")
-                .field("count", 2)
-                .endObject();
-        indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "2").setSource(source));
-        indexRandom(true, indexRequestBuilders);
-
-        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg")
-            .payload(Arrays.asList("title", "count"));
-        SearchResponse searchResponse = client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", prefix))
-            .execute().actionGet();
-        assertNoFailures(searchResponse);
-        CompletionSuggestion completionSuggestion = searchResponse.getSuggest().getSuggestion("foo");
-        List<CompletionSuggestion.Entry.Option> options = completionSuggestion.getEntries().get(0).getOptions();
-        assertThat(options.size(), equalTo(2));
-        assertThat(options.get(0).getText().toString(), equalTo("suggestion"));
-        assertThat(options.get(0).getScore(), equalTo(2f));
-        assertThat(options.get(1).getText().toString(), equalTo("suggest"));
-        assertThat(options.get(1).getScore(), equalTo(1f));
-
-        Map<String, List<Object>> firstPayload = options.get(0).getPayload();
-        assertThat(firstPayload.keySet(), containsInAnyOrder("title", "count"));
-        assertThat((String) firstPayload.get("title").get(0), equalTo("title2"));
-        assertThat((long) firstPayload.get("count").get(0), equalTo(2L));
-
-        Map<String, List<Object>> secondPayload = options.get(1).getPayload();
-        assertThat(secondPayload.keySet(), containsInAnyOrder("title", "count"));
-        assertThat((String) secondPayload.get("title").get(0), equalTo("title1"));
-        assertThat((long) secondPayload.get("count").get(0), equalTo(1L));
-    }
-
-    public void testSuggestWithPayload() throws Exception {
-        final CompletionMappingBuilder mapping = new CompletionMappingBuilder();
-        createIndexAndMapping(mapping);
-        int numDocs = randomIntBetween(10, 100);
-        int numPayloadFields = randomIntBetween(2, 5);
-        List<IndexRequestBuilder> indexRequestBuilders = new ArrayList<>();
-        for (int i = 1; i <= numDocs; i++) {
-            XContentBuilder source = jsonBuilder()
-                    .startObject()
-                    .startObject(FIELD)
-                    .field("input", "suggestion" + i)
-                    .field("weight", i)
-                    .endObject();
-            for (int j = 0; j < numPayloadFields; j++) {
-               source.field("test_field" + j, j + "value" + i);
-            }
-            source.endObject();
-            indexRequestBuilders.add(client().prepareIndex(INDEX, TYPE, "" + i).setSource(source));
-        }
-        indexRandom(true, indexRequestBuilders);
-
-        int suggestionSize = randomIntBetween(1, numDocs);
-        int numRequestedPayloadFields = randomIntBetween(2, numPayloadFields);
-        List<String> payloadFields = new ArrayList<>(numRequestedPayloadFields);
-        for (int i = 0; i < numRequestedPayloadFields; i++) {
-            payloadFields.add("test_field" + i + ".keyword");
-        }
-
-        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("sugg")
-            .size(suggestionSize).payload(payloadFields);
-        SearchResponse searchResponse = client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", prefix))
-            .execute().actionGet();
-        assertNoFailures(searchResponse);
-        CompletionSuggestion completionSuggestion = searchResponse.getSuggest().getSuggestion("foo");
-        CompletionSuggestion.Entry options = completionSuggestion.getEntries().get(0);
-        assertThat(options.getOptions().size(), equalTo(suggestionSize));
-        int id = numDocs;
-        for (CompletionSuggestion.Entry.Option option : options) {
-            assertThat(option.getText().toString(), equalTo("suggestion" + id));
-            assertThat(option.getPayload().size(), equalTo(numRequestedPayloadFields));
-            for (int i = 0; i < numRequestedPayloadFields; i++) {
-                List<Object> fieldValue = option.getPayload().get("test_field" + i + ".keyword");
-                assertNotNull(fieldValue);
-                assertThat(fieldValue.size(), equalTo(1));
-                assertThat((String)fieldValue.get(0), equalTo(i + "value" + id));
-            }
-            id--;
-        }
-    }
-
     public void testSuggestDocument() throws Exception {
         final CompletionMappingBuilder mapping = new CompletionMappingBuilder();
         createIndexAndMapping(mapping);

core/src/test/java/org/elasticsearch/search/suggest/completion/CompletionSuggesterBuilderTests.java

@@ -19,14 +19,10 @@
 
 package org.elasticsearch.search.suggest.completion;
 
-import com.carrotsearch.randomizedtesting.generators.RandomStrings;
-
-import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.search.suggest.AbstractSuggestionBuilderTestCase;
-import org.elasticsearch.search.suggest.SuggestBuilder;
 import org.elasticsearch.search.suggest.completion.context.CategoryQueryContext;
 import org.elasticsearch.search.suggest.completion.context.GeoQueryContext;
 
@@ -35,9 +31,7 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
-import static org.hamcrest.Matchers.containsString;
 
 public class CompletionSuggesterBuilderTests extends AbstractSuggestionBuilderTestCase<CompletionSuggestionBuilder> {
 
@@ -76,9 +70,6 @@ public class CompletionSuggesterBuilderTests extends AbstractSuggestionBuilderTe
                 testBuilder.regex(randomAsciiOfLength(10), RegexOptionsTests.randomRegexOptions());
                 break;
         }
-        List<String> payloads = new ArrayList<>();
-        Collections.addAll(payloads, generateRandomStringArray(5, 10, false, false));
-        maybeSet(testBuilder::payload, payloads);
         Map<String, List<? extends ToXContent>> contextMap = new HashMap<>();
         if (randomBoolean()) {
             int numContext = randomIntBetween(1, 5);
@@ -116,13 +107,8 @@ public class CompletionSuggesterBuilderTests extends AbstractSuggestionBuilderTe
 
     @Override
     protected void mutateSpecificParameters(CompletionSuggestionBuilder builder) throws IOException {
-        switch (randomIntBetween(0, 5)) {
+        switch (randomIntBetween(0, 4)) {
             case 0:
-                List<String> payloads = new ArrayList<>();
-                Collections.addAll(payloads, generateRandomStringArray(5, 10, false, false));
-                builder.payload(payloads);
-                break;
-            case 1:
                 int nCatContext = randomIntBetween(1, 5);
                 List<CategoryQueryContext> contexts = new ArrayList<>(nCatContext);
                 for (int i = 0; i < nCatContext; i++) {
@@ -130,7 +116,7 @@ public class CompletionSuggesterBuilderTests extends AbstractSuggestionBuilderTe
                 }
                 builder.contexts(Collections.singletonMap(randomAsciiOfLength(10), contexts));
                 break;
-            case 2:
+            case 1:
                 int nGeoContext = randomIntBetween(1, 5);
                 List<GeoQueryContext> geoContexts = new ArrayList<>(nGeoContext);
                 for (int i = 0; i < nGeoContext; i++) {
@@ -138,39 +124,17 @@ public class CompletionSuggesterBuilderTests extends AbstractSuggestionBuilderTe
                 }
                 builder.contexts(Collections.singletonMap(randomAsciiOfLength(10), geoContexts));
                 break;
-            case 3:
+            case 2:
                 builder.prefix(randomAsciiOfLength(10), FuzzyOptionsTests.randomFuzzyOptions());
                 break;
-            case 4:
+            case 3:
                 builder.prefix(randomAsciiOfLength(10), randomFrom(Fuzziness.ZERO, Fuzziness.ONE, Fuzziness.TWO));
                 break;
-            case 5:
+            case 4:
                 builder.regex(randomAsciiOfLength(10), RegexOptionsTests.randomRegexOptions());
                 break;
             default:
                 throw new IllegalStateException("should not through");
         }
     }
-
-    /**
-     * Test that a malformed JSON suggestion request fails.
-     */
-    public void testMalformedJsonRequestPayload() throws Exception {
-        final String field = RandomStrings.randomAsciiOfLength(random(), 10).toLowerCase(Locale.ROOT);
-        final String payload = "{\n" +
-                               "  \"bad-payload\" : { \n" +
-                               "    \"prefix\" : \"sug\",\n" +
-                               "    \"completion\" : { \n" +
-                               "      \"field\" : \"" + field + "\",\n " +
-                               "      \"payload\" : [ {\"payload\":\"field\"} ]\n" +
-                               "    }\n" +
-                               "  }\n" +
-                               "}\n";
-        try {
-            final SuggestBuilder suggestBuilder = SuggestBuilder.fromXContent(newParseContext(payload), suggesters);
-            fail("Should not have been able to create SuggestBuilder from malformed JSON: " + suggestBuilder);
-        } catch (ParsingException e) {
-            assertThat(e.getMessage(), containsString("failed to parse field [payload]"));
-        }
-    }
 }

core/src/test/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionTests.java

@@ -47,7 +47,7 @@ public class CompletionSuggestionTests extends ESTestCase {
         for (int i = 0; i < totalResults; i++) {
             Suggest.Suggestion<CompletionSuggestion.Entry> suggestion = randomFrom(shardSuggestions);
             suggestion.getEntries().get(0).addOption(new CompletionSuggestion.Entry.Option(i, new Text(""),
-                maxScore - i, Collections.emptyMap(), Collections.emptyMap()));
+                maxScore - i, Collections.emptyMap()));
         }
         CompletionSuggestion reducedSuggestion = CompletionSuggestion.reduceTo(shardSuggestions);
         assertNotNull(reducedSuggestion);

docs/reference/search/suggesters/completion-suggest.asciidoc

@@ -195,80 +195,11 @@ returns this response:
 // TESTRESPONSE
 
 The configured weight for a suggestion is returned as `_score`.
-The `text` field uses the `input` of your indexed suggestion. The document
-source is returned in `_source`. <<search-request-source-filtering, source filtering>>
+The `text` field uses the `input` of your indexed suggestion.
+Suggestions are document oriented, the document source is
+returned in `_source`. <<search-request-source-filtering, source filtering>>
 parameters are supported for filtering the document source.
 
-Suggestions are document oriented, you can specify fields to be
-returned as part of suggestion payload. All field types (`string`,
-`numeric`, `date`, etc) are supported.
-
-For example, if you index a "title" field along with the suggestion
-as follows:
-
-[source,js]
---------------------------------------------------
-PUT music/song/2?refresh
-{
-    "suggest" : "Nirvana",
-    "title" : "Nevermind"
-}
---------------------------------------------------
-// CONSOLE
-
-You can get the "title" as part of the suggestion
-payload by specifying it as a `payload`:
-
-[source,js]
---------------------------------------------------
-POST music/_suggest?pretty
-{
-    "song-suggest" : {
-        "prefix" : "n",
-        "completion" : {
-            "field" : "suggest",
-            "payload" : [ "title" ] <1>
-        }
-    }
-}
---------------------------------------------------
-// CONSOLE
-// TEST[continued]
-
-returns:
-
-[source,js]
---------------------------------------------------
-{
-  "_shards" : {
-    "total" : 5,
-    "successful" : 5,
-    "failed" : 0
-  },
-  "song-suggest" : [ {
-    "text" : "n",
-    "offset" : 0,
-    "length" : 1,
-    "options" : [ {
-      "text" : "Nirvana",
-      "_index": "music",
-      "_type": "song",
-      "_id": "2",
-      "_score" : 1.0,
-      "_source": {
-        "title": "Nevermind",
-        "suggest": "Nirvana"
-      },
-      "payload" : {
-        "title" : [ "Nevermind" ]
-      }
-    } ]
-  } ]
-}
---------------------------------------------------
-// TESTRESPONSE
-<1> The fields to be returned as part of each suggestion payload.
-
 The basic completion suggester query supports the following parameters:
 
 `field`:: The name of the field on which to run the query (required).

rest-api-spec/src/main/resources/rest-api-spec/test/suggest/20_completion.yaml

@@ -233,7 +233,7 @@ setup:
   - match:  { result.0.options.0.text: "baz" }
 
 ---
-"Suggestions with payload fields should work":
+"Suggestions with source should work":
 
   - do:
       index:
@@ -269,14 +269,17 @@ setup:
             text: "b"
             completion:
               field: suggest_6
-              payload: [ title, count ]
 
   - length: { result: 1  }
   - length: { result.0.options: 2  }
   - match:  { result.0.options.0.text: "baz" }
-  - match:  { result.0.options.0.payload.title: ["title_baz"] }
-  - match:  { result.0.options.0.payload.count: [3] }
+  - match:  { result.0.options.0._index: "test" }
+  - match:  { result.0.options.0._type: "test" }
+  - match:  { result.0.options.0._source.title: "title_baz" }
+  - match:  { result.0.options.0._source.count: 3 }
   - match:  { result.0.options.1.text: "bar" }
-  - match:  { result.0.options.1.payload.title: ["title_bar"] }
-  - match:  { result.0.options.1.payload.count: [4] }
+  - match:  { result.0.options.1._index: "test" }
+  - match:  { result.0.options.1._type: "test" }
+  - match:  { result.0.options.1._source.title: "title_bar" }
+  - match:  { result.0.options.1._source.count: 4 }