Return the _source of inner hit nested as is without wrapping it into its full path context

Due to a change happened via #26102 to make the nested source consistent
with or without source filtering, the _source of a nested inner hit was
always wrapped in the parent path. This turned out to be not ideal for
users relying on the nested source, as it would require additional parsing
on the client side. This change fixes this, the _source of nested inner hits
is now no longer wrapped by parent json objects, irregardless of whether
the _source is included as is or source filtering is used.

Internally source filtering and highlighting relies on the fact that the
_source of nested inner hits are accessible by its full field path, so
in order to now break this, the conversion of the _source into its binary
form is performed in FetchSourceSubPhase, after any potential source filtering
is performed to make sure the structure of _source of the nested inner hit
is consistent irregardless if source filtering is performed.

PR for #26944

Closes #26944

core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java

@@ -301,8 +301,6 @@ public class FetchPhase implements SearchPhase {
             }
             context.lookup().source().setSource(nestedSourceAsMap);
             XContentType contentType = tuple.v1();
-            BytesReference nestedSource = contentBuilder(contentType).map(nestedSourceAsMap).bytes();
-            context.lookup().source().setSource(nestedSource);
             context.lookup().source().setSourceContentType(contentType);
         }
         return new SearchHit(nestedTopDocId, uid.id(), documentMapper.typeText(), nestedIdentity, searchFields);

core/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourceSubPhase.java

@@ -20,13 +20,20 @@
 package org.elasticsearch.search.fetch.subphase;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.fetch.FetchSubPhase;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.lookup.SourceLookup;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Map;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.contentBuilder;
 
 public final class FetchSourceSubPhase implements FetchSubPhase {
 
@@ -35,22 +42,27 @@ public final class FetchSourceSubPhase implements FetchSubPhase {
         if (context.sourceRequested() == false) {
             return;
         }
+        final boolean nestedHit = hitContext.hit().getNestedIdentity() != null;
         SourceLookup source = context.lookup().source();
         FetchSourceContext fetchSourceContext = context.fetchSourceContext();
         assert fetchSourceContext.fetchSource();
-        if (fetchSourceContext.includes().length == 0 && fetchSourceContext.excludes().length == 0) {
-            hitContext.hit().sourceRef(source.internalSourceRef());
-            return;
+        if (nestedHit == false) {
+            if (fetchSourceContext.includes().length == 0 && fetchSourceContext.excludes().length == 0) {
+                hitContext.hit().sourceRef(source.internalSourceRef());
+                return;
+            }
+            if (source.internalSourceRef() == null) {
+                throw new IllegalArgumentException("unable to fetch fields from _source field: _source is disabled in the mappings " +
+                        "for index [" + context.indexShard().shardId().getIndexName() + "]");
+            }
         }
 
-        if (source.internalSourceRef() == null) {
-            throw new IllegalArgumentException("unable to fetch fields from _source field: _source is disabled in the mappings " +
-                    "for index [" + context.indexShard().shardId().getIndexName() + "]");
+        Object value = source.filter(fetchSourceContext);
+        if (nestedHit) {
+            value = getNestedSource((Map<String, Object>) value, hitContext);
         }
-
-        final Object value = source.filter(fetchSourceContext);
         try {
-            final int initialCapacity = Math.min(1024, source.internalSourceRef().length());
+            final int initialCapacity = nestedHit ? 1024 : Math.min(1024, source.internalSourceRef().length());
             BytesStreamOutput streamOutput = new BytesStreamOutput(initialCapacity);
             XContentBuilder builder = new XContentBuilder(source.sourceContentType().xContent(), streamOutput);
             builder.value(value);
@@ -58,6 +70,12 @@ public final class FetchSourceSubPhase implements FetchSubPhase {
         } catch (IOException e) {
             throw new ElasticsearchException("Error filtering source", e);
         }
+    }
 
+    private Map<String, Object> getNestedSource(Map<String, Object> sourceAsMap, HitContext hitContext) {
+        for (SearchHit.NestedIdentity o = hitContext.hit().getNestedIdentity(); o != null; o = o.getChild()) {
+            sourceAsMap = (Map<String, Object>) sourceAsMap.get(o.getField().string());
+        }
+        return sourceAsMap;
     }
 }

core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java

@@ -729,7 +729,7 @@ public class TopHitsIT extends ESIntegTestCase {
         assertThat(searchHits.getTotalHits(), equalTo(1L));
         assertThat(searchHits.getAt(0).getNestedIdentity().getField().string(), equalTo("comments"));
         assertThat(searchHits.getAt(0).getNestedIdentity().getOffset(), equalTo(0));
-        assertThat(extractValue("comments.date", searchHits.getAt(0).getSourceAsMap()), equalTo(1));
+        assertThat(extractValue("date", searchHits.getAt(0).getSourceAsMap()), equalTo(1));
 
         bucket = terms.getBucketByKey("b");
         assertThat(bucket.getDocCount(), equalTo(2L));
@@ -738,10 +738,10 @@ public class TopHitsIT extends ESIntegTestCase {
         assertThat(searchHits.getTotalHits(), equalTo(2L));
         assertThat(searchHits.getAt(0).getNestedIdentity().getField().string(), equalTo("comments"));
         assertThat(searchHits.getAt(0).getNestedIdentity().getOffset(), equalTo(1));
-        assertThat(extractValue("comments.date", searchHits.getAt(0).getSourceAsMap()), equalTo(2));
+        assertThat(extractValue("date", searchHits.getAt(0).getSourceAsMap()), equalTo(2));
         assertThat(searchHits.getAt(1).getNestedIdentity().getField().string(), equalTo("comments"));
         assertThat(searchHits.getAt(1).getNestedIdentity().getOffset(), equalTo(0));
-        assertThat(extractValue("comments.date", searchHits.getAt(1).getSourceAsMap()), equalTo(3));
+        assertThat(extractValue("date", searchHits.getAt(1).getSourceAsMap()), equalTo(3));
 
         bucket = terms.getBucketByKey("c");
         assertThat(bucket.getDocCount(), equalTo(1L));
@@ -750,7 +750,7 @@ public class TopHitsIT extends ESIntegTestCase {
         assertThat(searchHits.getTotalHits(), equalTo(1L));
         assertThat(searchHits.getAt(0).getNestedIdentity().getField().string(), equalTo("comments"));
         assertThat(searchHits.getAt(0).getNestedIdentity().getOffset(), equalTo(1));
-        assertThat(extractValue("comments.date", searchHits.getAt(0).getSourceAsMap()), equalTo(4));
+        assertThat(extractValue("date", searchHits.getAt(0).getSourceAsMap()), equalTo(4));
     }
 
     public void testTopHitsInSecondLayerNested() throws Exception {
@@ -803,49 +803,49 @@ public class TopHitsIT extends ESIntegTestCase {
         assertThat(topReviewers.getHits().getHits().length, equalTo(7));
 
         assertThat(topReviewers.getHits().getAt(0).getId(), equalTo("1"));
-        assertThat(extractValue("comments.reviewers.name", topReviewers.getHits().getAt(0).getSourceAsMap()), equalTo("user a"));
+        assertThat(extractValue("name", topReviewers.getHits().getAt(0).getSourceAsMap()), equalTo("user a"));
         assertThat(topReviewers.getHits().getAt(0).getNestedIdentity().getField().string(), equalTo("comments"));
         assertThat(topReviewers.getHits().getAt(0).getNestedIdentity().getOffset(), equalTo(0));
         assertThat(topReviewers.getHits().getAt(0).getNestedIdentity().getChild().getField().string(), equalTo("reviewers"));
         assertThat(topReviewers.getHits().getAt(0).getNestedIdentity().getChild().getOffset(), equalTo(0));
 
         assertThat(topReviewers.getHits().getAt(1).getId(), equalTo("1"));
-        assertThat(extractValue("comments.reviewers.name", topReviewers.getHits().getAt(1).getSourceAsMap()), equalTo("user b"));
+        assertThat(extractValue("name", topReviewers.getHits().getAt(1).getSourceAsMap()), equalTo("user b"));
         assertThat(topReviewers.getHits().getAt(1).getNestedIdentity().getField().string(), equalTo("comments"));
         assertThat(topReviewers.getHits().getAt(1).getNestedIdentity().getOffset(), equalTo(0));
         assertThat(topReviewers.getHits().getAt(1).getNestedIdentity().getChild().getField().string(), equalTo("reviewers"));
         assertThat(topReviewers.getHits().getAt(1).getNestedIdentity().getChild().getOffset(), equalTo(1));
 
         assertThat(topReviewers.getHits().getAt(2).getId(), equalTo("1"));
-        assertThat(extractValue("comments.reviewers.name", topReviewers.getHits().getAt(2).getSourceAsMap()), equalTo("user c"));
+        assertThat(extractValue("name", topReviewers.getHits().getAt(2).getSourceAsMap()), equalTo("user c"));
         assertThat(topReviewers.getHits().getAt(2).getNestedIdentity().getField().string(), equalTo("comments"));
         assertThat(topReviewers.getHits().getAt(2).getNestedIdentity().getOffset(), equalTo(0));
         assertThat(topReviewers.getHits().getAt(2).getNestedIdentity().getChild().getField().string(), equalTo("reviewers"));
         assertThat(topReviewers.getHits().getAt(2).getNestedIdentity().getChild().getOffset(), equalTo(2));
 
         assertThat(topReviewers.getHits().getAt(3).getId(), equalTo("1"));
-        assertThat(extractValue("comments.reviewers.name", topReviewers.getHits().getAt(3).getSourceAsMap()), equalTo("user c"));
+        assertThat(extractValue("name", topReviewers.getHits().getAt(3).getSourceAsMap()), equalTo("user c"));
         assertThat(topReviewers.getHits().getAt(3).getNestedIdentity().getField().string(), equalTo("comments"));
         assertThat(topReviewers.getHits().getAt(3).getNestedIdentity().getOffset(), equalTo(1));
         assertThat(topReviewers.getHits().getAt(3).getNestedIdentity().getChild().getField().string(), equalTo("reviewers"));
         assertThat(topReviewers.getHits().getAt(3).getNestedIdentity().getChild().getOffset(), equalTo(0));
 
         assertThat(topReviewers.getHits().getAt(4).getId(), equalTo("1"));
-        assertThat(extractValue("comments.reviewers.name", topReviewers.getHits().getAt(4).getSourceAsMap()), equalTo("user d"));
+        assertThat(extractValue("name", topReviewers.getHits().getAt(4).getSourceAsMap()), equalTo("user d"));
         assertThat(topReviewers.getHits().getAt(4).getNestedIdentity().getField().string(), equalTo("comments"));
         assertThat(topReviewers.getHits().getAt(4).getNestedIdentity().getOffset(), equalTo(1));
         assertThat(topReviewers.getHits().getAt(4).getNestedIdentity().getChild().getField().string(), equalTo("reviewers"));
         assertThat(topReviewers.getHits().getAt(4).getNestedIdentity().getChild().getOffset(), equalTo(1));
 
         assertThat(topReviewers.getHits().getAt(5).getId(), equalTo("1"));
-        assertThat(extractValue("comments.reviewers.name", topReviewers.getHits().getAt(5).getSourceAsMap()), equalTo("user e"));
+        assertThat(extractValue("name", topReviewers.getHits().getAt(5).getSourceAsMap()), equalTo("user e"));
         assertThat(topReviewers.getHits().getAt(5).getNestedIdentity().getField().string(), equalTo("comments"));
         assertThat(topReviewers.getHits().getAt(5).getNestedIdentity().getOffset(), equalTo(1));
         assertThat(topReviewers.getHits().getAt(5).getNestedIdentity().getChild().getField().string(), equalTo("reviewers"));
         assertThat(topReviewers.getHits().getAt(5).getNestedIdentity().getChild().getOffset(), equalTo(2));
 
         assertThat(topReviewers.getHits().getAt(6).getId(), equalTo("2"));
-        assertThat(extractValue("comments.reviewers.name", topReviewers.getHits().getAt(6).getSourceAsMap()), equalTo("user f"));
+        assertThat(extractValue("name", topReviewers.getHits().getAt(6).getSourceAsMap()), equalTo("user f"));
         assertThat(topReviewers.getHits().getAt(0).getNestedIdentity().getField().string(), equalTo("comments"));
         assertThat(topReviewers.getHits().getAt(0).getNestedIdentity().getOffset(), equalTo(0));
         assertThat(topReviewers.getHits().getAt(0).getNestedIdentity().getChild().getField().string(), equalTo("reviewers"));
@@ -901,7 +901,7 @@ public class TopHitsIT extends ESIntegTestCase {
         assertThat(field.getValue().toString(), equalTo("5"));
 
         assertThat(searchHit.getSourceAsMap().size(), equalTo(1));
-        assertThat(extractValue("comments.message", searchHit.getSourceAsMap()), equalTo("some comment"));
+        assertThat(extractValue("message", searchHit.getSourceAsMap()), equalTo("some comment"));
     }
 
     public void testTopHitsInNested() throws Exception {
@@ -934,7 +934,7 @@ public class TopHitsIT extends ESIntegTestCase {
             for (int j = 0; j < 3; j++) {
                 assertThat(searchHits.getAt(j).getNestedIdentity().getField().string(), equalTo("comments"));
                 assertThat(searchHits.getAt(j).getNestedIdentity().getOffset(), equalTo(0));
-                assertThat(extractValue("comments.id", searchHits.getAt(j).getSourceAsMap()), equalTo(0));
+                assertThat(extractValue("id", searchHits.getAt(j).getSourceAsMap()), equalTo(0));
 
                 HighlightField highlightField = searchHits.getAt(j).getHighlightFields().get("comments.message");
                 assertThat(highlightField.getFragments().length, equalTo(1));

core/src/test/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java

@@ -596,9 +596,9 @@ public class InnerHitsIT extends ESIntegTestCase {
         client().prepareIndex("index1", "message", "1").setSource(jsonBuilder().startObject()
                 .field("message", "quick brown fox")
                 .startArray("comments")
-                .startObject().field("message", "fox eat quick").endObject()
-                .startObject().field("message", "fox ate rabbit x y z").endObject()
-                .startObject().field("message", "rabbit got away").endObject()
+                .startObject().field("message", "fox eat quick").field("x", "y").endObject()
+                .startObject().field("message", "fox ate rabbit x y z").field("x", "y").endObject()
+                .startObject().field("message", "rabbit got away").field("x", "y").endObject()
                 .endArray()
                 .endObject()).get();
         refresh();
@@ -614,9 +614,11 @@ public class InnerHitsIT extends ESIntegTestCase {
         assertHitCount(response, 1);
 
         assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getTotalHits(), equalTo(2L));
-        assertThat(extractValue("comments.message", response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getSourceAsMap()),
+        assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getSourceAsMap().size(), equalTo(1));
+        assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getSourceAsMap().get("message"),
                 equalTo("fox eat quick"));
-        assertThat(extractValue("comments.message", response.getHits().getAt(0).getInnerHits().get("comments").getAt(1).getSourceAsMap()),
+        assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(1).getSourceAsMap().size(), equalTo(1));
+        assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(1).getSourceAsMap().get("message"),
                 equalTo("fox ate rabbit x y z"));
 
         response = client().prepareSearch()
@@ -627,9 +629,11 @@ public class InnerHitsIT extends ESIntegTestCase {
         assertHitCount(response, 1);
 
         assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getTotalHits(), equalTo(2L));
-        assertThat(extractValue("comments.message", response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getSourceAsMap()),
+        assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getSourceAsMap().size(), equalTo(2));
+        assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getSourceAsMap().get("message"),
                 equalTo("fox eat quick"));
-        assertThat(extractValue("comments.message", response.getHits().getAt(0).getInnerHits().get("comments").getAt(1).getSourceAsMap()),
+        assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getSourceAsMap().size(), equalTo(2));
+        assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(1).getSourceAsMap().get("message"),
                 equalTo("fox ate rabbit x y z"));
     }
 

docs/reference/aggregations/metrics/tophits-aggregation.asciidoc

@@ -329,10 +329,8 @@ Top hits response snippet with a nested hit, which resides in the first slot of
                     },
                     "_score": 0.2876821,
                     "_source": {
-                      "comments": {
-                        "comment": "This car could have better brakes", <3>
-                        "username": "baddriver007"
-                      }
+                      "comment": "This car could have better brakes", <3>
+                      "username": "baddriver007"
                     }
                   }
                 ]

docs/reference/search/request/inner-hits.asciidoc

@@ -158,10 +158,8 @@ An example of a response snippet that could be generated from the above search r
                   },
                   "_score": 1.0,
                   "_source": {
-                    "comments" : {
-                      "author": "nik9000",
-                      "number": 2
-                    }
+                    "author": "nik9000",
+                    "number": 2
                   }
                 }
               ]
@@ -406,12 +404,8 @@ Which would look like:
                   },
                   "_score": 0.6931472,
                   "_source": {
-                    "comments": {
-                      "votes": {
-                        "value": 1,
-                        "voter": "kimchy"
-                      }
-                    }
+                    "value": 1,
+                    "voter": "kimchy"
                   }
                 }
               ]