Add magic $_path stash key to docs tests (#24724)

Adds a "magic" key to the yaml testing stash mostly for use with
documentation tests. When unstashing an object, `$_path` is the
path into the current position in the object you are unstashing.
This means that in docs tests you can use
`// TESTRESPONSEs/somevalue/$body.${_path}/` to mean "replace
`somevalue` with whatever is the response in the same position."

Compare how you must carefully mock out all the numbers in the profile
response without this change:
```
// TESTRESPONSE[s/"id": "\[2aE02wS1R8q_QFnYu6vDVQ\]\[twitter\]\[1\]"/"id": $body.profile.shards.0.id/]
// TESTRESPONSE[s/"rewrite_time": 51443/"rewrite_time": $body.profile.shards.0.searches.0.rewrite_time/]
// TESTRESPONSE[s/"score": 51306/"score": $body.profile.shards.0.searches.0.query.0.breakdown.score/]
// TESTRESPONSE[s/"time_in_nanos": "1873811"/"time_in_nanos": $body.profile.shards.0.searches.0.query.0.time_in_nanos/]
// TESTRESPONSE[s/"build_scorer": 2935582/"build_scorer": $body.profile.shards.0.searches.0.query.0.breakdown.build_scorer/]
// TESTRESPONSE[s/"create_weight": 919297/"create_weight": $body.profile.shards.0.searches.0.query.0.breakdown.create_weight/]
// TESTRESPONSE[s/"next_doc": 53876/"next_doc": $body.profile.shards.0.searches.0.query.0.breakdown.next_doc/]
// TESTRESPONSE[s/"time_in_nanos": "391943"/"time_in_nanos": $body.profile.shards.0.searches.0.query.0.children.0.time_in_nanos/]
// TESTRESPONSE[s/"score": 28776/"score": $body.profile.shards.0.searches.0.query.0.children.0.breakdown.score/]
// TESTRESPONSE[s/"build_scorer": 784451/"build_scorer": $body.profile.shards.0.searches.0.query.0.children.0.breakdown.build_scorer/]
// TESTRESPONSE[s/"create_weight": 1669564/"create_weight": $body.profile.shards.0.searches.0.query.0.children.0.breakdown.create_weight/]
// TESTRESPONSE[s/"next_doc": 10111/"next_doc": $body.profile.shards.0.searches.0.query.0.children.0.breakdown.next_doc/]
// TESTRESPONSE[s/"time_in_nanos": "210682"/"time_in_nanos": $body.profile.shards.0.searches.0.query.0.children.1.time_in_nanos/]
// TESTRESPONSE[s/"score": 4552/"score": $body.profile.shards.0.searches.0.query.0.children.1.breakdown.score/]
// TESTRESPONSE[s/"build_scorer": 42602/"build_scorer": $body.profile.shards.0.searches.0.query.0.children.1.breakdown.build_scorer/]
// TESTRESPONSE[s/"create_weight": 89323/"create_weight": $body.profile.shards.0.searches.0.query.0.children.1.breakdown.create_weight/]
// TESTRESPONSE[s/"next_doc": 2852/"next_doc": $body.profile.shards.0.searches.0.query.0.children.1.breakdown.next_doc/]
// TESTRESPONSE[s/"time_in_nanos": "304311"/"time_in_nanos": $body.profile.shards.0.searches.0.collector.0.time_in_nanos/]
// TESTRESPONSE[s/"time_in_nanos": "32273"/"time_in_nanos": $body.profile.shards.0.searches.0.collector.0.children.0.time_in_nanos/]
```

To how you can cavalierly mock all the numbers at once with this change:
```
// TESTRESPONSE[s/(?<=[" ])\d+(\.\d+)?/$body.$_path/]
```

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy

@@ -168,6 +168,8 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
                 current.println("  - skip:")
                 current.println("      features: ")
                 current.println("        - stash_in_key")
+                current.println("        - stash_in_path")
+                current.println("        - stash_path_replace")
                 current.println("        - warnings")
             }
             if (test.skipTest) {

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy

@@ -90,6 +90,7 @@ public class SnippetsTask extends DefaultTask {
                      * tests cleaner.
                      */
                     subst = subst.replace('$body', '\\$body')
+                    subst = subst.replace('$_path', '\\$_path')
                     // \n is a new line....
                     subst = subst.replace('\\n', '\n')
                     snippet.contents = snippet.contents.replaceAll(

docs/README.asciidoc

@@ -47,6 +47,15 @@ for its modifiers:
   how it works. These are much more common than `// TEST[s/foo/bar]` because
   they are useful for eliding portions of the response that are not pertinent
   to the documentation.
+    * One interesting difference here is that you often want to match against
+    the response from Elasticsearch. To do that you can reference the "body" of
+    the response like this: `// TESTRESPONSE[s/"took": 25/"took": $body.took/]`.
+    Note the `$body` string. This says "I don't expect that 25 number in the
+    response, just match against what is in the response." Instead of writing
+    the path into the response after `$body` you can write `$_path` which
+    "figures out" the path. This is especially useful for making sweeping
+    assertions like "I made up all the numbers in this example, don't compare
+    them" which looks like `// TESTRESPONSE[s/\d+/$body.$_path/]`.
   * `// TESTRESPONSE[_cat]`: Add substitutions for testing `_cat` responses. Use
   this after all other substitutions so it doesn't make other substitutions
   difficult.

docs/reference/search/profile.asciidoc

@@ -51,7 +51,7 @@ This will yield the following result:
    "profile": {
      "shards": [
         {
-           "id": "[2aE02wS1R8q_QFnYu6vDVQ][twitter][1]",
+           "id": "[2aE02wS1R8q_QFnYu6vDVQ][twitter][0]",
            "searches": [
               {
                  "query": [
@@ -139,27 +139,9 @@ This will yield the following result:
 }
 --------------------------------------------------
 // TESTRESPONSE[s/"took": 25/"took": $body.took/]
-// TESTRESPONSE[s/"hits": \[...\]/"hits": $body.hits.hits/]
-// TESTRESPONSE[s/"id": "\[2aE02wS1R8q_QFnYu6vDVQ\]\[twitter\]\[1\]"/"id": $body.profile.shards.0.id/]
-// TESTRESPONSE[s/"rewrite_time": 51443/"rewrite_time": $body.profile.shards.0.searches.0.rewrite_time/]
-// TESTRESPONSE[s/"score": 51306/"score": $body.profile.shards.0.searches.0.query.0.breakdown.score/]
-// TESTRESPONSE[s/"time_in_nanos": "1873811"/"time_in_nanos": $body.profile.shards.0.searches.0.query.0.time_in_nanos/]
-// TESTRESPONSE[s/"build_scorer": 2935582/"build_scorer": $body.profile.shards.0.searches.0.query.0.breakdown.build_scorer/]
-// TESTRESPONSE[s/"create_weight": 919297/"create_weight": $body.profile.shards.0.searches.0.query.0.breakdown.create_weight/]
-// TESTRESPONSE[s/"next_doc": 53876/"next_doc": $body.profile.shards.0.searches.0.query.0.breakdown.next_doc/]
-// TESTRESPONSE[s/"time_in_nanos": "391943"/"time_in_nanos": $body.profile.shards.0.searches.0.query.0.children.0.time_in_nanos/]
-// TESTRESPONSE[s/"score": 28776/"score": $body.profile.shards.0.searches.0.query.0.children.0.breakdown.score/]
-// TESTRESPONSE[s/"build_scorer": 784451/"build_scorer": $body.profile.shards.0.searches.0.query.0.children.0.breakdown.build_scorer/]
-// TESTRESPONSE[s/"create_weight": 1669564/"create_weight": $body.profile.shards.0.searches.0.query.0.children.0.breakdown.create_weight/]
-// TESTRESPONSE[s/"next_doc": 10111/"next_doc": $body.profile.shards.0.searches.0.query.0.children.0.breakdown.next_doc/]
-// TESTRESPONSE[s/"time_in_nanos": "210682"/"time_in_nanos": $body.profile.shards.0.searches.0.query.0.children.1.time_in_nanos/]
-// TESTRESPONSE[s/"score": 4552/"score": $body.profile.shards.0.searches.0.query.0.children.1.breakdown.score/]
-// TESTRESPONSE[s/"build_scorer": 42602/"build_scorer": $body.profile.shards.0.searches.0.query.0.children.1.breakdown.build_scorer/]
-// TESTRESPONSE[s/"create_weight": 89323/"create_weight": $body.profile.shards.0.searches.0.query.0.children.1.breakdown.create_weight/]
-// TESTRESPONSE[s/"next_doc": 2852/"next_doc": $body.profile.shards.0.searches.0.query.0.children.1.breakdown.next_doc/]
-// TESTRESPONSE[s/"time_in_nanos": "304311"/"time_in_nanos": $body.profile.shards.0.searches.0.collector.0.time_in_nanos/]
-// TESTRESPONSE[s/"time_in_nanos": "32273"/"time_in_nanos": $body.profile.shards.0.searches.0.collector.0.children.0.time_in_nanos/]
-// Sorry for this mess....
+// TESTRESPONSE[s/"hits": \[...\]/"hits": $body.$_path/]
+// TESTRESPONSE[s/(?<=[" ])\d+(\.\d+)?/$body.$_path/]
+// TESTRESPONSE[s/\[2aE02wS1R8q_QFnYu6vDVQ\]\[twitter\]\[0\]/$body.$_path/]
 
 <1> Search results are returned, but were omitted here for brevity
 
@@ -174,7 +156,7 @@ First, the overall structure of the profile response is as follows:
    "profile": {
         "shards": [
            {
-              "id": "[2aE02wS1R8q_QFnYu6vDVQ][twitter][1]",  <1>
+              "id": "[2aE02wS1R8q_QFnYu6vDVQ][twitter][0]",  <1>
               "searches": [
                  {
                     "query": [...],             <2>
@@ -189,10 +171,10 @@ First, the overall structure of the profile response is as follows:
 }
 --------------------------------------------------
 // TESTRESPONSE[s/"profile": /"took": $body.took, "timed_out": $body.timed_out, "_shards": $body._shards, "hits": $body.hits, "profile": /]
-// TESTRESPONSE[s/"id": "\[2aE02wS1R8q_QFnYu6vDVQ\]\[twitter\]\[1\]"/"id": $body.profile.shards.0.id/]
-// TESTRESPONSE[s/"query": \[...\]/"query": $body.profile.shards.0.searches.0.query/]
-// TESTRESPONSE[s/"rewrite_time": 51443/"rewrite_time": $body.profile.shards.0.searches.0.rewrite_time/]
-// TESTRESPONSE[s/"collector": \[...\]/"collector": $body.profile.shards.0.searches.0.collector/]
+// TESTRESPONSE[s/(?<=[" ])\d+(\.\d+)?/$body.$_path/]
+// TESTRESPONSE[s/\[2aE02wS1R8q_QFnYu6vDVQ\]\[twitter\]\[0\]/$body.$_path/]
+// TESTRESPONSE[s/"query": \[...\]/"query": $body.$_path/]
+// TESTRESPONSE[s/"collector": \[...\]/"collector": $body.$_path/]
 // TESTRESPONSE[s/"aggregations": \[...\]/"aggregations": []/]
 <1> A profile is returned for each shard that participated in the response, and is identified
 by a unique ID
@@ -267,11 +249,10 @@ The overall structure of this query tree will resemble your original Elasticsear
     }
 ]
 --------------------------------------------------
-// TESTRESPONSE[s/^/{\n"took": $body.took,\n"timed_out": $body.timed_out,\n"_shards": $body._shards,\n"hits": $body.hits,\n"profile": {\n"shards": [ {\n"id": "$body.profile.shards.0.id",\n"searches": [{\n/]
-// TESTRESPONSE[s/]$/],"rewrite_time": $body.profile.shards.0.searches.0.rewrite_time, "collector": $body.profile.shards.0.searches.0.collector}], "aggregations": []}]}}/]
-// TESTRESPONSE[s/"time_in_nanos": "1873811",\n.+"breakdown": \{...\}/"time_in_nanos": $body.profile.shards.0.searches.0.query.0.time_in_nanos, "breakdown": $body.profile.shards.0.searches.0.query.0.breakdown/]
-// TESTRESPONSE[s/"time_in_nanos": "391943",\n.+"breakdown": \{...\}/"time_in_nanos": $body.profile.shards.0.searches.0.query.0.children.0.time_in_nanos, "breakdown": $body.profile.shards.0.searches.0.query.0.children.0.breakdown/]
-// TESTRESPONSE[s/"time_in_nanos": "210682",\n.+"breakdown": \{...\}/"time_in_nanos": $body.profile.shards.0.searches.0.query.0.children.1.time_in_nanos, "breakdown": $body.profile.shards.0.searches.0.query.0.children.1.breakdown/]
+// TESTRESPONSE[s/^/{\n"took": $body.took,\n"timed_out": $body.timed_out,\n"_shards": $body._shards,\n"hits": $body.hits,\n"profile": {\n"shards": [ {\n"id": "$body.$_path",\n"searches": [{\n/]
+// TESTRESPONSE[s/]$/],"rewrite_time": $body.$_path, "collector": $body.$_path}], "aggregations": []}]}}/]
+// TESTRESPONSE[s/(?<=[" ])\d+(\.\d+)?/$body.$_path/]
+// TESTRESPONSE[s/"breakdown": \{...\}/"breakdown": $body.$_path/]
 <1> The breakdown timings are omitted for simplicity
 
 Based on the profile structure, we can see that our `match` query was rewritten by Lucene into a BooleanQuery with two
@@ -309,13 +290,9 @@ The `breakdown` component lists detailed timing statistics about low-level Lucen
    "advance_count": 0
 }
 --------------------------------------------------
-// TESTRESPONSE[s/^/{\n"took": $body.took,\n"timed_out": $body.timed_out,\n"_shards": $body._shards,\n"hits": $body.hits,\n"profile": {\n"shards": [ {\n"id": "$body.profile.shards.0.id",\n"searches": [{\n"query": [{\n"type": "BooleanQuery",\n"description": "message:message message:number",\n"time_in_nanos": $body.profile.shards.0.searches.0.query.0.time_in_nanos,/]
-// TESTRESPONSE[s/}$/},\n"children": $body.profile.shards.0.searches.0.query.0.children}],\n"rewrite_time": $body.profile.shards.0.searches.0.rewrite_time, "collector": $body.profile.shards.0.searches.0.collector}], "aggregations": []}]}}/]
-// TESTRESPONSE[s/"score": 51306/"score": $body.profile.shards.0.searches.0.query.0.breakdown.score/]
-// TESTRESPONSE[s/"time_in_nanos": "1873811"/"time_in_nanos": $body.profile.shards.0.searches.0.query.0.time_in_nanos/]
-// TESTRESPONSE[s/"build_scorer": 2935582/"build_scorer": $body.profile.shards.0.searches.0.query.0.breakdown.build_scorer/]
-// TESTRESPONSE[s/"create_weight": 919297/"create_weight": $body.profile.shards.0.searches.0.query.0.breakdown.create_weight/]
-// TESTRESPONSE[s/"next_doc": 53876/"next_doc": $body.profile.shards.0.searches.0.query.0.breakdown.next_doc/]
+// TESTRESPONSE[s/^/{\n"took": $body.took,\n"timed_out": $body.timed_out,\n"_shards": $body._shards,\n"hits": $body.hits,\n"profile": {\n"shards": [ {\n"id": "$body.$_path",\n"searches": [{\n"query": [{\n"type": "BooleanQuery",\n"description": "message:message message:number",\n"time_in_nanos": $body.$_path,/]
+// TESTRESPONSE[s/}$/},\n"children": $body.$_path}],\n"rewrite_time": $body.$_path, "collector": $body.$_path}], "aggregations": []}]}}/]
+// TESTRESPONSE[s/(?<=[" ])\d+(\.\d+)?/$body.$_path/]
 
 Timings are listed in wall-clock nanoseconds and are not normalized at all.  All caveats about the overall
 `time_in_nanos` apply here.  The intention of the breakdown is to give you a feel for A) what machinery in Lucene is
@@ -416,10 +393,9 @@ Looking at the previous example:
    }
 ]
 --------------------------------------------------
-// TESTRESPONSE[s/^/{\n"took": $body.took,\n"timed_out": $body.timed_out,\n"_shards": $body._shards,\n"hits": $body.hits,\n"profile": {\n"shards": [ {\n"id": "$body.profile.shards.0.id",\n"searches": [{\n"query": $body.profile.shards.0.searches.0.query,\n"rewrite_time": $body.profile.shards.0.searches.0.rewrite_time,/]
+// TESTRESPONSE[s/^/{\n"took": $body.took,\n"timed_out": $body.timed_out,\n"_shards": $body._shards,\n"hits": $body.hits,\n"profile": {\n"shards": [ {\n"id": "$body.$_path",\n"searches": [{\n"query": $body.$_path,\n"rewrite_time": $body.$_path,/]
 // TESTRESPONSE[s/]$/]}], "aggregations": []}]}}/]
-// TESTRESPONSE[s/"time_in_nanos": "304311"/"time_in_nanos": $body.profile.shards.0.searches.0.collector.0.time_in_nanos/]
-// TESTRESPONSE[s/"time_in_nanos": "32273"/"time_in_nanos": $body.profile.shards.0.searches.0.collector.0.children.0.time_in_nanos/]
+// TESTRESPONSE[s/(?<=[" ])\d+(\.\d+)?/$body.$_path/]
 
 We see a single collector named `SimpleTopScoreDocCollector` wrapped into `CancellableCollector`. `SimpleTopScoreDocCollector` is the default "scoring and sorting"
 `Collector` used by Elasticsearch.  The `reason` field attempts to give a plain english description of the class name.  The

test/framework/src/main/java/org/elasticsearch/test/rest/yaml/Features.java

@@ -41,6 +41,7 @@ public final class Features {
             "headers",
             "stash_in_key",
             "stash_in_path",
+            "stash_path_replace",
             "warnings",
             "yaml"));
 

test/framework/src/main/java/org/elasticsearch/test/rest/yaml/Stash.java

@@ -28,9 +28,9 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.TreeMap;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -40,6 +40,7 @@ import java.util.regex.Pattern;
  */
 public class Stash implements ToXContent {
     private static final Pattern EXTENDED_KEY = Pattern.compile("\\$\\{([^}]+)\\}");
+    private static final Pattern PATH = Pattern.compile("\\$_path");
 
     private static final Logger logger = Loggers.getLogger(Stash.class);
 
@@ -125,19 +126,22 @@ public class Stash implements ToXContent {
      */
     @SuppressWarnings("unchecked") // Safe because we check that all the map keys are string in unstashObject
     public Map<String, Object> replaceStashedValues(Map<String, Object> map) throws IOException {
-        return (Map<String, Object>) unstashObject(map);
+        return (Map<String, Object>) unstashObject(new ArrayList<>(), map);
     }
 
-    private Object unstashObject(Object obj) throws IOException {
+    private Object unstashObject(List<Object> path, Object obj) throws IOException {
         if (obj instanceof List) {
             List<?> list = (List<?>) obj;
             List<Object> result = new ArrayList<>();
+            int index = 0;
             for (Object o : list) {
+                path.add(index++);
                 if (containsStashedValue(o)) {
-                    result.add(getValue(o.toString()));
+                    result.add(getValue(path, o.toString()));
                 } else {
-                    result.add(unstashObject(o));
+                    result.add(unstashObject(path, o));
                 }
+                path.remove(path.size() - 1);
             }
             return result;
         }
@@ -150,11 +154,13 @@ public class Stash implements ToXContent {
                 if (containsStashedValue(key)) {
                     key = getValue(key).toString();
                 }
+                path.add(key);
                 if (containsStashedValue(value)) {
-                    value = getValue(value.toString());
+                    value = getValue(path, value.toString());
                 } else {
-                    value = unstashObject(value);
+                    value = unstashObject(path, value);
                 }
+                path.remove(path.size() - 1);
                 if (null != result.putIfAbsent(key, value)) {
                     throw new IllegalArgumentException("Unstashing has caused a key conflict! The map is [" + result + "] and the key is ["
                             + entry.getKey() + "] which unstashes to [" + key + "]");
@@ -165,6 +171,34 @@ public class Stash implements ToXContent {
         return obj;
     }
 
+    /**
+     * Lookup a value from the stash adding support for a special key ({@code $_path}) which
+     * returns a string that is the location in the path of the of the object currently being
+     * unstashed. This is useful during documentation testing.
+     */
+    private Object getValue(List<Object> path, String key) throws IOException {
+        Matcher matcher = PATH.matcher(key);
+        if (false == matcher.find()) {
+            return getValue(key);
+        }
+        StringBuilder pathBuilder = new StringBuilder();
+        Iterator<Object> element = path.iterator();
+        if (element.hasNext()) {
+            pathBuilder.append(element.next());
+            while (element.hasNext()) {
+                pathBuilder.append('.');
+                pathBuilder.append(element.next());
+            }
+        }
+        String builtPath = Matcher.quoteReplacement(pathBuilder.toString());
+        StringBuffer newKey = new StringBuffer(key.length());
+        do {
+            matcher.appendReplacement(newKey, builtPath);
+        } while (matcher.find());
+        matcher.appendTail(newKey);
+        return getValue(newKey.toString());
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.field("stash", stash);

test/framework/src/test/java/org/elasticsearch/test/rest/yaml/StashTests.java

@@ -95,7 +95,6 @@ public class StashTests extends ESTestCase {
                             + key + "] which unstashes to [foobar]");
     }
 
-
     public void testReplaceStashedValuesStashKeyInList() throws IOException {
         Stash stash = new Stash();
         stash.stashValue("stashed", "bar");
@@ -117,4 +116,43 @@ public class StashTests extends ESTestCase {
         assertEquals(expected, actual);
         assertThat(actual, not(sameInstance(map)));
     }
+
+    public void testPathInList() throws IOException {
+        Stash stash = new Stash();
+        stash.stashValue("body", singletonMap("foo", Arrays.asList("a", "b")));
+
+        Map<String, Object> expected;
+        Map<String, Object> map;
+        if (randomBoolean()) {
+            expected = singletonMap("foo", Arrays.asList("test", "boooooh!"));
+            map = singletonMap("foo", Arrays.asList("test", "${body.$_path}oooooh!"));
+        } else {
+            expected = singletonMap("foo", Arrays.asList("test", "b"));
+            map = singletonMap("foo", Arrays.asList("test", "$body.$_path"));
+        }
+
+        Map<String, Object> actual = stash.replaceStashedValues(map);
+        assertEquals(expected, actual);
+        assertThat(actual, not(sameInstance(map)));
+    }
+
+    public void testPathInMapValue() throws IOException {
+        Stash stash = new Stash();
+        stash.stashValue("body", singletonMap("foo", singletonMap("a", "b")));
+
+        Map<String, Object> expected;
+        Map<String, Object> map;
+        if (randomBoolean()) {
+            expected = singletonMap("foo", singletonMap("a", "boooooh!"));
+            map = singletonMap("foo", singletonMap("a", "${body.$_path}oooooh!"));
+        } else {
+            expected = singletonMap("foo", singletonMap("a", "b"));
+            map = singletonMap("foo", singletonMap("a", "$body.$_path"));
+        }
+
+        Map<String, Object> actual = stash.replaceStashedValues(map);
+        assertEquals(expected, actual);
+        assertThat(actual, not(sameInstance(map)));
+    }
+
 }