Log YAML test file on failure (#91349)

Relates #91081

docs/reference/cluster/nodes-stats.asciidoc

@@ -2747,7 +2747,6 @@ response.
 --------------------------------------------------
 GET /_nodes/stats?metric=ingest&filter_path=nodes.*.ingest
 --------------------------------------------------
-// TESTRESPONSE[skip:"AwaitsFix https://github.com/elastic/elasticsearch/issues/91081"]
 
 To further refine the response, change the `filter_path` value.
 For example, the following request only returns ingest pipeline statistics.

test/yaml-rest-runner/build.gradle

@@ -1,5 +1,6 @@
-
 apply plugin: 'elasticsearch.build'
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.yaml-rest-compat-test'
 
 dependencies {
   api project(':test:framework')

test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java

@@ -53,6 +53,7 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Optional;
 import java.util.Set;
 import java.util.SortedSet;
 import java.util.TreeSet;
@@ -498,6 +499,15 @@ public abstract class ESClientYamlSuiteTestCase extends ESRestTestCase {
         try {
             executableSection.execute(restTestExecutionContext);
         } catch (AssertionError | Exception e) {
+            // Dump the original yaml file, if available, for reference.
+            Optional<Path> file = testCandidate.getRestTestSuite().getFile();
+            if (file.isPresent()) {
+                try {
+                    logger.info("Dump test yaml [{}] on failure:\n{}", file.get(), Files.readString(file.get()));
+                } catch (IOException ex) {
+                    logger.info("Did not dump test yaml [{}] on failure due to an exception [{}]", file.get(), ex);
+                }
+            }
             // Dump the stash on failure. Instead of dumping it in true json we escape `\n`s so stack traces are easier to read
             logger.info(
                 "Stash dump on test failure [{}]",

test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java

@@ -27,6 +27,7 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.TreeSet;
 import java.util.stream.Collectors;
@@ -67,13 +68,13 @@ public class ClientYamlTestSuite {
                 Files.newInputStream(file)
             )
         ) {
-            return parse(api, filename, parser);
+            return parse(api, filename, Optional.of(file), parser);
         } catch (Exception e) {
             throw new IOException("Error parsing " + api + "/" + filename, e);
         }
     }
 
-    public static ClientYamlTestSuite parse(String api, String suiteName, XContentParser parser) throws IOException {
+    public static ClientYamlTestSuite parse(String api, String suiteName, Optional<Path> file, XContentParser parser) throws IOException {
         if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
             throw new XContentParseException(
                 parser.getTokenLocation(),
@@ -99,11 +100,12 @@ public class ClientYamlTestSuite {
             }
         }
 
-        return new ClientYamlTestSuite(api, suiteName, setupSection, teardownSection, new ArrayList<>(testSections));
+        return new ClientYamlTestSuite(api, suiteName, file, setupSection, teardownSection, new ArrayList<>(testSections));
     }
 
     private final String api;
     private final String name;
+    private final Optional<Path> file;
     private final SetupSection setupSection;
     private final TeardownSection teardownSection;
     private final List<ClientYamlTestSection> testSections;
@@ -111,12 +113,14 @@ public class ClientYamlTestSuite {
     public ClientYamlTestSuite(
         String api,
         String name,
+        Optional<Path> file,
         SetupSection setupSection,
         TeardownSection teardownSection,
         List<ClientYamlTestSection> testSections
     ) {
         this.api = api.replace("\\", "/");  // since api's are sourced from the filesystem normalize backslashes to "/"
         this.name = name;
+        this.file = file;
         this.setupSection = Objects.requireNonNull(setupSection, "setup section cannot be null");
         this.teardownSection = Objects.requireNonNull(teardownSection, "teardown section cannot be null");
         this.testSections = Collections.unmodifiableList(testSections);
@@ -134,6 +138,10 @@ public class ClientYamlTestSuite {
         return api + "/" + name;
     }
 
+    public Optional<Path> getFile() {
+        return file;
+    }
+
     public SetupSection getSetupSection() {
         return setupSection;
     }

test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuiteTests.java

@@ -14,11 +14,14 @@ import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.xcontent.XContentLocation;
 import org.elasticsearch.xcontent.yaml.YamlXContent;
 
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Optional;
 import java.util.regex.Pattern;
 
 import static java.util.Collections.emptyList;
@@ -82,10 +85,11 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars
               - match: {test_type.properties.text.analyzer: whitespace}
             """);
 
-        ClientYamlTestSuite restTestSuite = ClientYamlTestSuite.parse(getTestClass().getName(), getTestName(), parser);
+        ClientYamlTestSuite restTestSuite = ClientYamlTestSuite.parse(getTestClass().getName(), getTestName(), Optional.empty(), parser);
 
         assertThat(restTestSuite, notNullValue());
         assertThat(restTestSuite.getName(), equalTo(getTestName()));
+        assertThat(restTestSuite.getFile().isPresent(), equalTo(false));
         assertThat(restTestSuite.getSetupSection(), notNullValue());
         if (includeSetup) {
             assertThat(restTestSuite.getSetupSection().isEmpty(), equalTo(false));
@@ -196,10 +200,11 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars
              - match:   { _version: 1}
              - match:   { _source: { foo: bar }}""");
 
-        ClientYamlTestSuite restTestSuite = ClientYamlTestSuite.parse(getTestClass().getName(), getTestName(), parser);
+        ClientYamlTestSuite restTestSuite = ClientYamlTestSuite.parse(getTestClass().getName(), getTestName(), Optional.empty(), parser);
 
         assertThat(restTestSuite, notNullValue());
         assertThat(restTestSuite.getName(), equalTo(getTestName()));
+        assertThat(restTestSuite.getFile().isPresent(), equalTo(false));
 
         assertThat(restTestSuite.getSetupSection().isEmpty(), equalTo(true));
 
@@ -308,10 +313,11 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars
                         params:       { bar: 'xxx' }
             """);
 
-        ClientYamlTestSuite restTestSuite = ClientYamlTestSuite.parse(getTestClass().getName(), getTestName(), parser);
+        ClientYamlTestSuite restTestSuite = ClientYamlTestSuite.parse(getTestClass().getName(), getTestName(), Optional.empty(), parser);
 
         assertThat(restTestSuite, notNullValue());
         assertThat(restTestSuite.getName(), equalTo(getTestName()));
+        assertThat(restTestSuite.getFile().isPresent(), equalTo(false));
 
         assertThat(restTestSuite.getSetupSection().isEmpty(), equalTo(true));
 
@@ -383,7 +389,7 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars
 
         Exception e = expectThrows(
             ParsingException.class,
-            () -> ClientYamlTestSuite.parse(getTestClass().getName(), getTestName(), parser)
+            () -> ClientYamlTestSuite.parse(getTestClass().getName(), getTestName(), Optional.empty(), parser)
         );
         assertThat(e.getMessage(), containsString("duplicate test section"));
     }
@@ -406,10 +412,11 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars
               - match: {test_type.properties.text.analyzer: whitespace}
             """);
 
-        ClientYamlTestSuite restTestSuite = ClientYamlTestSuite.parse(getTestClass().getName(), getTestName(), parser);
+        ClientYamlTestSuite restTestSuite = ClientYamlTestSuite.parse(getTestClass().getName(), getTestName(), Optional.empty(), parser);
 
         assertThat(restTestSuite, notNullValue());
         assertThat(restTestSuite.getName(), equalTo(getTestName()));
+        assertThat(restTestSuite.getFile().isPresent(), equalTo(false));
         assertThat(restTestSuite.getTestSections().size(), equalTo(1));
 
         assertThat(restTestSuite.getTestSections().get(0).getName(), equalTo("Broken on some os"));
@@ -422,6 +429,49 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars
         assertThat(restTestSuite.getTestSections().get(0).getSkipSection().getFeatures(), containsInAnyOrder("skip_os"));
     }
 
+    public void testParseFileWithSingleTestSection() throws Exception {
+        final Path filePath = createTempFile("tyf", ".yml");
+        Files.writeString(filePath, """
+            ---
+            "Index with ID":
+
+             - do:
+                  index:
+                      index:  test-weird-index-中文
+                      type:   weird.type
+                      id:     1
+                      body:   { foo: bar }
+
+             - is_true:   ok""" + "\n");
+
+        ClientYamlTestSuite restTestSuite = ClientYamlTestSuite.parse(ExecutableSection.XCONTENT_REGISTRY, "api", filePath);
+
+        assertThat(restTestSuite, notNullValue());
+        assertThat(
+            restTestSuite.getName(),
+            equalTo(filePath.getFileName().toString().substring(0, filePath.getFileName().toString().lastIndexOf('.')))
+        );
+        assertThat(restTestSuite.getFile().isPresent(), equalTo(true));
+        assertThat(restTestSuite.getFile().get(), equalTo(filePath));
+
+        assertThat(restTestSuite.getSetupSection().isEmpty(), equalTo(true));
+
+        assertThat(restTestSuite.getTestSections().size(), equalTo(1));
+
+        assertThat(restTestSuite.getTestSections().get(0).getName(), equalTo("Index with ID"));
+        assertThat(restTestSuite.getTestSections().get(0).getSkipSection().isEmpty(), equalTo(true));
+        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().size(), equalTo(2));
+        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(0), instanceOf(DoSection.class));
+        DoSection doSection = (DoSection) restTestSuite.getTestSections().get(0).getExecutableSections().get(0);
+        assertThat(doSection.getCatch(), nullValue());
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("index"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3));
+        assertThat(doSection.getApiCallSection().hasBody(), equalTo(true));
+        assertThat(restTestSuite.getTestSections().get(0).getExecutableSections().get(1), instanceOf(IsTrueAssertion.class));
+        IsTrueAssertion trueAssertion = (IsTrueAssertion) restTestSuite.getTestSections().get(0).getExecutableSections().get(1);
+        assertThat(trueAssertion.getField(), equalTo("ok"));
+    }
+
     public void testAddingDoWithoutSkips() {
         int lineNumber = between(1, 10000);
         DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
@@ -435,6 +485,7 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars
         ClientYamlTestSuite clientYamlTestSuite = new ClientYamlTestSuite(
             "api",
             "name",
+            Optional.empty(),
             SetupSection.EMPTY,
             TeardownSection.EMPTY,
             Collections.singletonList(section)
@@ -582,7 +633,14 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars
         }
         sections.add(new ClientYamlTestSection(new XContentLocation(0, 0), "section2", SkipSection.EMPTY, doSections));
 
-        ClientYamlTestSuite testSuite = new ClientYamlTestSuite("api", "name", SetupSection.EMPTY, TeardownSection.EMPTY, sections);
+        ClientYamlTestSuite testSuite = new ClientYamlTestSuite(
+            "api",
+            "name",
+            Optional.empty(),
+            SetupSection.EMPTY,
+            TeardownSection.EMPTY,
+            sections
+        );
         Exception e = expectThrows(IllegalArgumentException.class, testSuite::validate);
         assertEquals(String.format(Locale.ROOT, """
             api/name:
@@ -714,6 +772,13 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars
             }
             default -> throw new UnsupportedOperationException();
         }
-        return new ClientYamlTestSuite("api", "name", setupSection, teardownSection, Collections.singletonList(clientYamlTestSection));
+        return new ClientYamlTestSuite(
+            "api",
+            "name",
+            Optional.empty(),
+            setupSection,
+            teardownSection,
+            Collections.singletonList(clientYamlTestSection)
+        );
     }
 }

test/yaml-rest-runner/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCaseFailLogIT.java

@@ -0,0 +1,76 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.rest.yaml;
+
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.test.MockLogAppender;
+import org.elasticsearch.test.junit.annotations.TestLogging;
+
+import java.io.IOException;
+
+public class ESClientYamlSuiteTestCaseFailLogIT extends ESClientYamlSuiteTestCase {
+
+    public ESClientYamlSuiteTestCaseFailLogIT(final ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return createParameters();
+    }
+
+    @TestLogging(
+        reason = "testing logging on yaml test failure",
+        value = "org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCaseFailIT:INFO"
+    )
+    @Override
+    public void test() throws IOException {
+        final MockLogAppender mockLogAppender = new MockLogAppender();
+        try {
+            mockLogAppender.start();
+            Loggers.addAppender(LogManager.getLogger(ESClientYamlSuiteTestCaseFailLogIT.class), mockLogAppender);
+
+            mockLogAppender.addExpectation(
+                new MockLogAppender.SeenEventExpectation(
+                    "message with dump of the test yaml",
+                    ESClientYamlSuiteTestCaseFailLogIT.class.getCanonicalName(),
+                    Level.INFO,
+                    "Dump test yaml [*10_fail.yml] on failure:*Hello: testid*Hello: test2id*"
+                )
+            );
+
+            mockLogAppender.addExpectation(
+                new MockLogAppender.SeenEventExpectation(
+                    "message with stash dump of response",
+                    ESClientYamlSuiteTestCaseFailLogIT.class.getCanonicalName(),
+                    Level.INFO,
+                    "Stash dump on test failure [{*Hello: testid*}]"
+                )
+            );
+
+            try {
+                super.test();
+            } catch (AssertionError error) {
+                // If it is the error we expect, ignore it, else re-throw.
+                if (error.getMessage().contains("foo: expected \"Hello: test2id\" but was \"Hello: testid\"") == false) {
+                    throw error;
+                }
+            }
+
+            mockLogAppender.assertAllExpectationsMatched();
+        } finally {
+            Loggers.removeAppender(LogManager.getLogger(ESClientYamlSuiteTestCaseFailLogIT.class), mockLogAppender);
+            mockLogAppender.stop();
+        }
+    }
+}

test/yaml-rest-runner/src/yamlRestTest/resources/rest-api-spec/test/10_fail.yml

@@ -0,0 +1,17 @@
+---
+"Basic YAML integration test that fails":
+
+  - do:
+      index:
+        index: test_1
+        id:    testid
+        body:  { "foo": "Hello: testid" }
+
+  - do:
+      get:
+        index: test_1
+        id:    testid
+
+  - match: { _index:   test_1 }
+  - match: { _id:      testid      }
+  - match: { _source:  { foo: "Hello: test2id" } }

x-pack/qa/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java

@@ -168,6 +168,7 @@ public abstract class CoreTestTranslater {
             modified = new ClientYamlTestSuite(
                 candidate.getApi(),
                 candidate.getName(),
+                candidate.getRestTestSuite().getFile(),
                 new SetupSection(candidate.getSetupSection().getSkipSection(), setup),
                 candidate.getTeardownSection(),
                 List.of()