Highlight fields in request order

Because json objects are unordered this also adds an explicit order syntax
that looks like
    "highlight": {
        "fields": [
            {"title":{ /*params*/ }},
            {"text":{ /*params*/ }}
        ]
    }

This is not useful for any of the builtin highlighters but will be useful
in plugins.

Closes #4649

docs/reference/search/request/highlighting.asciidoc

@@ -547,3 +547,20 @@ keep in mind that scoring more phrases consumes more time and memory.
 
 If using `matched_fields` keep in mind that `phrase_limit` phrases per
 matched field are considered.
+
+[[explicit-field-order]]
+=== Field Highlight Order
+Elasticsearch highlights the fields in the order that they are sent.  Per the
+json spec objects are unordered but if you need to be explicit about the order
+that fields are highlighted then you can use an array for `fields` like this:
+[source,js]
+--------------------------------------------------
+    "highlight": {
+        "fields": [
+            {"title":{ /*params*/ }},
+            {"text":{ /*params*/ }}
+        ]
+    }
+--------------------------------------------------
+None of the highlighters built into Elasticsearch care about the order that the
+fields are highlighted but a plugin may.

src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java

@@ -806,6 +806,15 @@ public class SearchRequestBuilder extends ActionRequestBuilder<SearchRequest, Se
         return this;
     }
 
+    /**
+     * Send the fields to be highlighted using a syntax that is specific about the order in which they should be highlighted.
+     * @return this for chaining
+     */
+    public SearchRequestBuilder setHighlighterExplicitFieldOrder(boolean explicitFieldOrder) {
+        highlightBuilder().useExplicitFieldOrder(explicitFieldOrder);
+        return this;
+    }
+
     /**
      * Delegates to {@link org.elasticsearch.search.suggest.SuggestBuilder#setText(String)}.
      */

src/main/java/org/elasticsearch/search/highlight/HighlightBuilder.java

@@ -74,6 +74,8 @@ public class HighlightBuilder implements ToXContent {
 
     private Boolean forceSource;
 
+    private boolean useExplicitFieldOrder = false;
+
     /**
      * Adds a field to be highlighted with default fragment size of 100 characters, and
      * default number of fragments of 5 using the default encoder
@@ -289,6 +291,15 @@ public class HighlightBuilder implements ToXContent {
         return this;
     }
 
+    /**
+     * Send the fields to be highlighted using a syntax that is specific about the order in which they should be highlighted.
+     * @return this for chaining
+     */
+    public HighlightBuilder useExplicitFieldOrder(boolean useExplicitFieldOrder) {
+        this.useExplicitFieldOrder = useExplicitFieldOrder;
+        return this;
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject("highlight");
@@ -347,8 +358,15 @@ public class HighlightBuilder implements ToXContent {
             builder.field("force_source", forceSource);
         }
         if (fields != null) {
-            builder.startObject("fields");
+            if (useExplicitFieldOrder) {
+                builder.startArray("fields");
+            } else {
+                builder.startObject("fields");
+            }
             for (Field field : fields) {
+                if (useExplicitFieldOrder) {
+                    builder.startObject();
+                }
                 builder.startObject(field.name());
                 if (field.preTags != null) {
                     builder.field("pre_tags", field.preTags);
@@ -406,10 +424,16 @@ public class HighlightBuilder implements ToXContent {
                 }
 
                 builder.endObject();
+                if (useExplicitFieldOrder) {
+                    builder.endObject();
+                }
+            }
+            if (useExplicitFieldOrder) {
+                builder.endArray();
+            } else {
+                builder.endObject();
             }
-            builder.endObject();
         }
-
         builder.endObject();
         return builder;
     }

src/main/java/org/elasticsearch/search/highlight/HighlighterParseElement.java

@@ -28,6 +28,7 @@ import org.elasticsearch.search.SearchParseElement;
 import org.elasticsearch.search.SearchParseException;
 import org.elasticsearch.search.internal.SearchContext;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.Set;
 
@@ -92,6 +93,24 @@ public class HighlighterParseElement implements SearchParseElement {
                         postTagsList.add(parser.text());
                     }
                     globalOptionsBuilder.postTags(postTagsList.toArray(new String[postTagsList.size()]));
+                } else if ("fields".equals(topLevelFieldName)) {
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                        if (token == XContentParser.Token.START_OBJECT) {
+                            String highlightFieldName = null;
+                            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                                if (token == XContentParser.Token.FIELD_NAME) {
+                                    if (highlightFieldName != null) {
+                                        throw new SearchParseException(context, "If highlighter fields is an array it must contain objects containing a single field");
+                                    }
+                                    highlightFieldName = parser.currentName();
+                                } else if (token == XContentParser.Token.START_OBJECT) {
+                                    fieldsOptions.add(Tuple.tuple(highlightFieldName, parseFields(parser, context)));
+                                }
+                            }
+                        } else {
+                            throw new SearchParseException(context, "If highlighter fields is an array it must contain objects containing a single field");
+                        }
+                    }
                 }
             } else if (token.isValue()) {
                 if ("order".equals(topLevelFieldName)) {
@@ -141,73 +160,7 @@ public class HighlighterParseElement implements SearchParseElement {
                         if (token == XContentParser.Token.FIELD_NAME) {
                             highlightFieldName = parser.currentName();
                         } else if (token == XContentParser.Token.START_OBJECT) {
-                            SearchContextHighlight.FieldOptions.Builder fieldOptionsBuilder = new SearchContextHighlight.FieldOptions.Builder();
-                            String fieldName = null;
-                            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                                if (token == XContentParser.Token.FIELD_NAME) {
-                                    fieldName = parser.currentName();
-                                } else if (token == XContentParser.Token.START_ARRAY) {
-                                    if ("pre_tags".equals(fieldName) || "preTags".equals(fieldName)) {
-                                        List<String> preTagsList = Lists.newArrayList();
-                                        while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                                            preTagsList.add(parser.text());
-                                        }
-                                        fieldOptionsBuilder.preTags(preTagsList.toArray(new String[preTagsList.size()]));
-                                    } else if ("post_tags".equals(fieldName) || "postTags".equals(fieldName)) {
-                                        List<String> postTagsList = Lists.newArrayList();
-                                        while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                                            postTagsList.add(parser.text());
-                                        }
-                                        fieldOptionsBuilder.postTags(postTagsList.toArray(new String[postTagsList.size()]));
-                                    } else if ("matched_fields".equals(fieldName) || "matchedFields".equals(fieldName)) {
-                                        Set<String> matchedFields = Sets.newHashSet();
-                                        while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                                            matchedFields.add(parser.text());
-                                        }
-                                        fieldOptionsBuilder.matchedFields(matchedFields);
-                                    }
-                                } else if (token.isValue()) {
-                                    if ("fragment_size".equals(fieldName) || "fragmentSize".equals(fieldName)) {
-                                        fieldOptionsBuilder.fragmentCharSize(parser.intValue());
-                                    } else if ("number_of_fragments".equals(fieldName) || "numberOfFragments".equals(fieldName)) {
-                                        fieldOptionsBuilder.numberOfFragments(parser.intValue());
-                                    } else if ("fragment_offset".equals(fieldName) || "fragmentOffset".equals(fieldName)) {
-                                        fieldOptionsBuilder.fragmentOffset(parser.intValue());
-                                    } else if ("highlight_filter".equals(fieldName) || "highlightFilter".equals(fieldName)) {
-                                        fieldOptionsBuilder.highlightFilter(parser.booleanValue());
-                                    } else if ("order".equals(fieldName)) {
-                                        fieldOptionsBuilder.scoreOrdered("score".equals(parser.text()));
-                                    } else if ("require_field_match".equals(fieldName) || "requireFieldMatch".equals(fieldName)) {
-                                        fieldOptionsBuilder.requireFieldMatch(parser.booleanValue());
-                                    } else if ("boundary_max_scan".equals(topLevelFieldName) || "boundaryMaxScan".equals(topLevelFieldName)) {
-                                        fieldOptionsBuilder.boundaryMaxScan(parser.intValue());
-                                    } else if ("boundary_chars".equals(topLevelFieldName) || "boundaryChars".equals(topLevelFieldName)) {
-                                        char[] charsArr = parser.text().toCharArray();
-                                        Character[] boundaryChars = new Character[charsArr.length];
-                                        for (int i = 0; i < charsArr.length; i++) {
-                                            boundaryChars[i] = charsArr[i];
-                                        }
-                                        fieldOptionsBuilder.boundaryChars(boundaryChars);
-                                    } else if ("type".equals(fieldName)) {
-                                        fieldOptionsBuilder.highlighterType(parser.text());
-                                    } else if ("fragmenter".equals(fieldName)) {
-                                        fieldOptionsBuilder.fragmenter(parser.text());
-                                    } else if ("no_match_size".equals(fieldName) || "noMatchSize".equals(fieldName)) {
-                                        fieldOptionsBuilder.noMatchSize(parser.intValue());
-                                    } else if ("force_source".equals(fieldName) || "forceSource".equals(fieldName)) {
-                                        fieldOptionsBuilder.forceSource(parser.booleanValue());
-                                    } else if ("phrase_limit".equals(fieldName) || "phraseLimit".equals(fieldName)) {
-                                        fieldOptionsBuilder.phraseLimit(parser.intValue());
-                                    }
-                                } else if (token == XContentParser.Token.START_OBJECT) {
-                                    if ("highlight_query".equals(fieldName) || "highlightQuery".equals(fieldName)) {
-                                        fieldOptionsBuilder.highlightQuery(context.queryParserService().parse(parser).query());
-                                    } else if ("options".equals(fieldName)) {
-                                        fieldOptionsBuilder.options(parser.map());
-                                    }
-                                }
-                            }
-                            fieldsOptions.add(Tuple.tuple(highlightFieldName, fieldOptionsBuilder));
+                            fieldsOptions.add(Tuple.tuple(highlightFieldName, parseFields(parser, context)));
                         }
                     }
                 } else if ("highlight_query".equals(topLevelFieldName) || "highlightQuery".equals(topLevelFieldName)) {
@@ -229,4 +182,76 @@ public class HighlighterParseElement implements SearchParseElement {
 
         context.highlight(new SearchContextHighlight(fields));
     }
+
+    private SearchContextHighlight.FieldOptions.Builder parseFields(XContentParser parser, SearchContext context) throws IOException {
+        XContentParser.Token token;
+
+        SearchContextHighlight.FieldOptions.Builder fieldOptionsBuilder = new SearchContextHighlight.FieldOptions.Builder();
+        String fieldName = null;
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                fieldName = parser.currentName();
+            } else if (token == XContentParser.Token.START_ARRAY) {
+                if ("pre_tags".equals(fieldName) || "preTags".equals(fieldName)) {
+                    List<String> preTagsList = Lists.newArrayList();
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                        preTagsList.add(parser.text());
+                    }
+                    fieldOptionsBuilder.preTags(preTagsList.toArray(new String[preTagsList.size()]));
+                } else if ("post_tags".equals(fieldName) || "postTags".equals(fieldName)) {
+                    List<String> postTagsList = Lists.newArrayList();
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                        postTagsList.add(parser.text());
+                    }
+                    fieldOptionsBuilder.postTags(postTagsList.toArray(new String[postTagsList.size()]));
+                } else if ("matched_fields".equals(fieldName) || "matchedFields".equals(fieldName)) {
+                    Set<String> matchedFields = Sets.newHashSet();
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                        matchedFields.add(parser.text());
+                    }
+                    fieldOptionsBuilder.matchedFields(matchedFields);
+                }
+            } else if (token.isValue()) {
+                if ("fragment_size".equals(fieldName) || "fragmentSize".equals(fieldName)) {
+                    fieldOptionsBuilder.fragmentCharSize(parser.intValue());
+                } else if ("number_of_fragments".equals(fieldName) || "numberOfFragments".equals(fieldName)) {
+                    fieldOptionsBuilder.numberOfFragments(parser.intValue());
+                } else if ("fragment_offset".equals(fieldName) || "fragmentOffset".equals(fieldName)) {
+                    fieldOptionsBuilder.fragmentOffset(parser.intValue());
+                } else if ("highlight_filter".equals(fieldName) || "highlightFilter".equals(fieldName)) {
+                    fieldOptionsBuilder.highlightFilter(parser.booleanValue());
+                } else if ("order".equals(fieldName)) {
+                    fieldOptionsBuilder.scoreOrdered("score".equals(parser.text()));
+                } else if ("require_field_match".equals(fieldName) || "requireFieldMatch".equals(fieldName)) {
+                    fieldOptionsBuilder.requireFieldMatch(parser.booleanValue());
+                } else if ("boundary_max_scan".equals(fieldName) || "boundaryMaxScan".equals(fieldName)) {
+                    fieldOptionsBuilder.boundaryMaxScan(parser.intValue());
+                } else if ("boundary_chars".equals(fieldName) || "boundaryChars".equals(fieldName)) {
+                    char[] charsArr = parser.text().toCharArray();
+                    Character[] boundaryChars = new Character[charsArr.length];
+                    for (int i = 0; i < charsArr.length; i++) {
+                        boundaryChars[i] = charsArr[i];
+                    }
+                    fieldOptionsBuilder.boundaryChars(boundaryChars);
+                } else if ("type".equals(fieldName)) {
+                    fieldOptionsBuilder.highlighterType(parser.text());
+                } else if ("fragmenter".equals(fieldName)) {
+                    fieldOptionsBuilder.fragmenter(parser.text());
+                } else if ("no_match_size".equals(fieldName) || "noMatchSize".equals(fieldName)) {
+                    fieldOptionsBuilder.noMatchSize(parser.intValue());
+                } else if ("force_source".equals(fieldName) || "forceSource".equals(fieldName)) {
+                    fieldOptionsBuilder.forceSource(parser.booleanValue());
+                } else if ("phrase_limit".equals(fieldName) || "phraseLimit".equals(fieldName)) {
+                    fieldOptionsBuilder.phraseLimit(parser.intValue());
+                }
+            } else if (token == XContentParser.Token.START_OBJECT) {
+                if ("highlight_query".equals(fieldName) || "highlightQuery".equals(fieldName)) {
+                    fieldOptionsBuilder.highlightQuery(context.queryParserService().parse(parser).query());
+                } else if ("options".equals(fieldName)) {
+                    fieldOptionsBuilder.options(parser.map());
+                }
+            }
+        }
+        return fieldOptionsBuilder;
+    }
 }

src/main/java/org/elasticsearch/search/highlight/SearchContextHighlight.java

@@ -35,7 +35,7 @@ public class SearchContextHighlight {
 
     public SearchContextHighlight(Collection<Field> fields) {
         assert fields != null;
-        this.fields = Maps.newHashMap();
+        this.fields = new LinkedHashMap<String, Field>(fields.size());
         for (Field field : fields) {
             this.fields.put(field.field, field);
         }

src/test/java/org/elasticsearch/search/highlight/CustomHighlighter.java

@@ -23,6 +23,7 @@ import org.elasticsearch.common.text.StringText;
 import org.elasticsearch.common.text.Text;
 
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 
 /**
@@ -38,9 +39,24 @@ public class CustomHighlighter implements Highlighter {
     @Override
     public HighlightField highlight(HighlighterContext highlighterContext) {
         SearchContextHighlight.Field field = highlighterContext.field;
+        CacheEntry cacheEntry = (CacheEntry) highlighterContext.hitContext.cache().get("test-custom");
+        if (cacheEntry == null) {
+            cacheEntry = new CacheEntry();
+            highlighterContext.hitContext.cache().put("test-custom", cacheEntry);
+            cacheEntry.docId = highlighterContext.hitContext.docId();
+            cacheEntry.position = 1;
+        } else {
+            if (cacheEntry.docId == highlighterContext.hitContext.docId()) {
+                cacheEntry.position++;
+            } else {
+                cacheEntry.docId = highlighterContext.hitContext.docId();
+                cacheEntry.position = 1;
+            }
+        }
 
         List<Text> responses = Lists.newArrayList();
-        responses.add(new StringText("standard response"));
+        responses.add(new StringText(String.format(Locale.ENGLISH, "standard response for %s at position %s", field.field(),
+                cacheEntry.position)));
 
         if (field.fieldOptions().options() != null) {
             for (Map.Entry<String, Object> entry : field.fieldOptions().options().entrySet()) {
@@ -50,4 +66,9 @@ public class CustomHighlighter implements Highlighter {
 
         return new HighlightField(highlighterContext.fieldName, responses.toArray(new Text[]{}));
     }
+
+    private static class CacheEntry {
+        private int position;
+        private int docId;
+    }
 }

src/test/java/org/elasticsearch/search/highlight/CustomHighlighterSearchTests.java

@@ -20,11 +20,11 @@ package org.elasticsearch.search.highlight;
 
 import com.google.common.collect.Maps;
 import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.test.ElasticsearchIntegrationTest;
+import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
+import org.elasticsearch.test.ElasticsearchIntegrationTest.Scope;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -32,8 +32,6 @@ import java.io.IOException;
 import java.util.Map;
 
 import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder;
-import static org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
-import static org.elasticsearch.test.ElasticsearchIntegrationTest.Scope;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHighlight;
 import static org.hamcrest.Matchers.equalTo;
 
@@ -53,12 +51,12 @@ public class CustomHighlighterSearchTests extends ElasticsearchIntegrationTest {
 
     @Before
     protected void setup() throws Exception{
-        client().prepareIndex("test", "test", "1").setSource(XContentFactory.jsonBuilder()
-                .startObject()
-                .field("name", "arbitrary content")
-                .endObject())
-                .setRefresh(true).execute().actionGet();
-        client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForYellowStatus().execute().actionGet();
+        indexRandom(true,
+                client().prepareIndex("test", "test", "1").setSource(
+                        "name", "arbitrary content", "other_name", "foo", "other_other_name", "bar"),
+                client().prepareIndex("test", "test", "2").setSource(
+                        "other_name", "foo", "other_other_name", "bar"));
+        ensureYellow();
     }
 
     @Test
@@ -67,7 +65,7 @@ public class CustomHighlighterSearchTests extends ElasticsearchIntegrationTest {
                 .setQuery(QueryBuilders.matchAllQuery())
                 .addHighlightedField("name").setHighlighterType("test-custom")
                 .execute().actionGet();
-        assertHighlight(searchResponse, 0, "name", 0, equalTo("standard response"));
+        assertHighlight(searchResponse, 0, "name", 0, equalTo("standard response for name at position 1"));
     }
 
     @Test
@@ -83,7 +81,7 @@ public class CustomHighlighterSearchTests extends ElasticsearchIntegrationTest {
                 .addHighlightedField(highlightConfig)
                 .execute().actionGet();
 
-        assertHighlight(searchResponse, 0, "name", 0, equalTo("standard response"));
+        assertHighlight(searchResponse, 0, "name", 0, equalTo("standard response for name at position 1"));
         assertHighlight(searchResponse, 0, "name", 1, equalTo("field:myFieldOption:someValue"));
     }
 
@@ -99,7 +97,27 @@ public class CustomHighlighterSearchTests extends ElasticsearchIntegrationTest {
                 .addHighlightedField("name")
                 .execute().actionGet();
 
-        assertHighlight(searchResponse, 0, "name", 0, equalTo("standard response"));
+        assertHighlight(searchResponse, 0, "name", 0, equalTo("standard response for name at position 1"));
         assertHighlight(searchResponse, 0, "name", 1, equalTo("field:myGlobalOption:someValue"));
     }
+
+    @Test
+    public void testThatCustomHighlighterReceivesFieldsInOrder() throws Exception {
+        SearchResponse searchResponse = client().prepareSearch("test").setTypes("test")
+                .setQuery(QueryBuilders.boolQuery().must(QueryBuilders.matchAllQuery()).should(QueryBuilders
+                        .termQuery("name", "arbitrary")))
+                .setHighlighterType("test-custom")
+                .addHighlightedField("name")
+                .addHighlightedField("other_name")
+                .addHighlightedField("other_other_name")
+                .setHighlighterExplicitFieldOrder(true)
+                .get();
+
+        assertHighlight(searchResponse, 0, "name", 0, equalTo("standard response for name at position 1"));
+        assertHighlight(searchResponse, 0, "other_name", 0, equalTo("standard response for other_name at position 2"));
+        assertHighlight(searchResponse, 0, "other_other_name", 0, equalTo("standard response for other_other_name at position 3"));
+        assertHighlight(searchResponse, 1, "name", 0, equalTo("standard response for name at position 1"));
+        assertHighlight(searchResponse, 1, "other_name", 0, equalTo("standard response for other_name at position 2"));
+        assertHighlight(searchResponse, 1, "other_other_name", 0, equalTo("standard response for other_other_name at position 3"));
+    }
 }