[TEST] improve validation of yaml suites (#34957)

Validation of test sections and suites consists of checking that the proper skip features sections are in place depending on the features used in tests.

The validation logic was previously only performed on do sections included in each test section, and the skip needed to be present in the same test section. What happens often though is that the skip is added to the setup section, or the teardown section.

This commit improves the validation of test suites by validating setup and teardown section first, then looking at each test section while still eventually reading the skip section from setup or teardown.

We are also making SkipSection, SetupSection, TearDownSection, ClientYamlTestSection and ClientYamlTestSuite immutable. Previously it was possible to utilize constants like SetupSection.EMPTY, which were modifiable and affect every other future users by modifiying them. This has been corrected.

Also, validation has been improved to cumulate errors so that all the errors from a suite will be listed at once.

Relates to #34735

test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java

@@ -38,7 +38,6 @@ import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestApi;
 import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestSpec;
 import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection;
 import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSuite;
-import org.elasticsearch.test.rest.yaml.section.DoSection;
 import org.elasticsearch.test.rest.yaml.section.ExecutableSection;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -184,19 +183,44 @@ public abstract class ESClientYamlSuiteTestCase extends ESRestTestCase {
      */
     public static Iterable<Object[]> createParameters(NamedXContentRegistry executeableSectionRegistry) throws Exception {
         String[] paths = resolvePathsProperty(REST_TESTS_SUITE, ""); // default to all tests under the test root
-        List<Object[]> tests = new ArrayList<>();
         Map<String, Set<Path>> yamlSuites = loadSuites(paths);
+        List<ClientYamlTestSuite> suites = new ArrayList<>();
+        IllegalArgumentException validationException = null;
         // yaml suites are grouped by directory (effectively by api)
         for (String api : yamlSuites.keySet()) {
             List<Path> yamlFiles = new ArrayList<>(yamlSuites.get(api));
             for (Path yamlFile : yamlFiles) {
-                ClientYamlTestSuite restTestSuite = ClientYamlTestSuite.parse(executeableSectionRegistry, api, yamlFile);
-                for (ClientYamlTestSection testSection : restTestSuite.getTestSections()) {
-                    tests.add(new Object[]{ new ClientYamlTestCandidate(restTestSuite, testSection) });
+                ClientYamlTestSuite suite = ClientYamlTestSuite.parse(executeableSectionRegistry, api, yamlFile);
+                suites.add(suite);
+                try {
+                    suite.validate();
+                } catch(IllegalArgumentException e) {
+                    if (validationException == null) {
+                        validationException = new IllegalArgumentException("Validation errors for the following test suites:\n- "
+                            + e.getMessage());
+                    } else {
+                        String previousMessage = validationException.getMessage();
+                        Throwable[] suppressed = validationException.getSuppressed();
+                        validationException = new IllegalArgumentException(previousMessage + "\n- " + e.getMessage());
+                        for (Throwable t : suppressed) {
+                            validationException.addSuppressed(t);
+                        }
+                    }
+                    validationException.addSuppressed(e);
                 }
             }
         }
 
+        if (validationException != null) {
+            throw validationException;
+        }
+
+        List<Object[]> tests = new ArrayList<>();
+        for (ClientYamlTestSuite yamlTestSuite : suites) {
+            for (ClientYamlTestSection testSection : yamlTestSuite.getTestSections()) {
+                tests.add(new Object[]{ new ClientYamlTestCandidate(yamlTestSuite, testSection) });
+            }
+        }
         //sort the candidates so they will always be in the same order before being shuffled, for repeatability
         tests.sort(Comparator.comparing(o -> ((ClientYamlTestCandidate) o[0]).getTestPath()));
         return tests;
@@ -361,7 +385,7 @@ public abstract class ESClientYamlSuiteTestCase extends ESRestTestCase {
             }
         } finally {
             logger.debug("start teardown test [{}]", testCandidate.getTestPath());
-            for (DoSection doSection : testCandidate.getTeardownSection().getDoSections()) {
+            for (ExecutableSection doSection : testCandidate.getTeardownSection().getDoSections()) {
                 executeSection(doSection);
             }
             logger.debug("end teardown test [{}]", testCandidate.getTestPath());

test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java

@@ -18,14 +18,15 @@
  */
 package org.elasticsearch.test.rest.yaml.section;
 
-import org.elasticsearch.client.NodeSelector;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.xcontent.XContentLocation;
 import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
+import java.util.Objects;
 
 /**
  * Represents a test section, which is composed of a skip section and multiple executable sections.
@@ -33,34 +34,37 @@ import java.util.List;
 public class ClientYamlTestSection implements Comparable<ClientYamlTestSection> {
     public static ClientYamlTestSection parse(XContentParser parser) throws IOException {
         ParserUtils.advanceToFieldName(parser);
-        ClientYamlTestSection testSection = new ClientYamlTestSection(parser.getTokenLocation(), parser.currentName());
+        XContentLocation sectionLocation = parser.getTokenLocation();
+        String sectionName = parser.currentName();
+        List<ExecutableSection> executableSections = new ArrayList<>();
         try {
             parser.nextToken();
-            testSection.setSkipSection(SkipSection.parseIfNext(parser));
+            SkipSection skipSection = SkipSection.parseIfNext(parser);
             while (parser.currentToken() != XContentParser.Token.END_ARRAY) {
                 ParserUtils.advanceToFieldName(parser);
-                testSection.addExecutableSection(ExecutableSection.parse(parser));
+                executableSections.add(ExecutableSection.parse(parser));
             }
             if (parser.nextToken() != XContentParser.Token.END_OBJECT) {
-                throw new IllegalArgumentException("malformed section [" + testSection.getName() + "] expected ["
+                throw new IllegalArgumentException("malformed section [" + sectionName + "] expected ["
                         + XContentParser.Token.END_OBJECT + "] but was [" + parser.currentToken() + "]");
             }
             parser.nextToken();
-            return testSection;
+            return new ClientYamlTestSection(sectionLocation, sectionName, skipSection, executableSections);
         } catch (Exception e) {
-            throw new ParsingException(parser.getTokenLocation(), "Error parsing test named [" + testSection.getName() + "]", e);
+            throw new ParsingException(parser.getTokenLocation(), "Error parsing test named [" + sectionName + "]", e);
         }
     }
 
     private final XContentLocation location;
     private final String name;
-    private SkipSection skipSection;
+    private final SkipSection skipSection;
     private final List<ExecutableSection> executableSections;
 
-    public ClientYamlTestSection(XContentLocation location, String name) {
+    ClientYamlTestSection(XContentLocation location, String name, SkipSection skipSection, List<ExecutableSection> executableSections) {
         this.location = location;
         this.name = name;
-        this.executableSections = new ArrayList<>();
+        this.skipSection = Objects.requireNonNull(skipSection, "skip section cannot be null");
+        this.executableSections = Collections.unmodifiableList(executableSections);
     }
 
     public XContentLocation getLocation() {
@@ -75,40 +79,10 @@ public class ClientYamlTestSection implements Comparable<ClientYamlTestSection>
         return skipSection;
     }
 
-    public void setSkipSection(SkipSection skipSection) {
-        this.skipSection = skipSection;
-    }
-
     public List<ExecutableSection> getExecutableSections() {
         return executableSections;
     }
 
-    public void addExecutableSection(ExecutableSection executableSection) {
-        if (executableSection instanceof DoSection) {
-            DoSection doSection = (DoSection) executableSection;
-            if (false == doSection.getExpectedWarningHeaders().isEmpty()
-                    && false == skipSection.getFeatures().contains("warnings")) {
-                throw new IllegalArgumentException("Attempted to add a [do] with a [warnings] section without a corresponding [skip] so "
-                        + "runners that do not support the [warnings] section can skip the test at line ["
-                        + doSection.getLocation().lineNumber + "]");
-            }
-            if (NodeSelector.ANY != doSection.getApiCallSection().getNodeSelector()
-                    && false == skipSection.getFeatures().contains("node_selector")) {
-                throw new IllegalArgumentException("Attempted to add a [do] with a [node_selector] section without a corresponding "
-                    + "[skip] so runners that do not support the [node_selector] section can skip the test at line ["
-                    + doSection.getLocation().lineNumber + "]");
-            }
-        }
-        if (executableSection instanceof ContainsAssertion) {
-            if (false == skipSection.getFeatures().contains("contains")) {
-                throw new IllegalArgumentException("Attempted to add a [contains] assertion without a corresponding "
-                    + "[skip: \"features\": \"contains\"] so runners that do not support the [contains] assertion " +
-                    "can skip the test at line [" + executableSection.getLocation().lineNumber + "]");
-            }
-        }
-        this.executableSections.add(executableSection);
-    }
-
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;

test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java

@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.test.rest.yaml.section;
 
+import org.elasticsearch.client.NodeSelector;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
@@ -32,9 +33,13 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
+import java.util.Objects;
 import java.util.Set;
 import java.util.TreeSet;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 /**
  * Holds a REST test suite loaded from a specific yaml file.
@@ -79,11 +84,10 @@ public class ClientYamlTestSuite {
                     "expected token to be START_OBJECT but was " + parser.currentToken());
         }
 
-        ClientYamlTestSuite restTestSuite = new ClientYamlTestSuite(api, suiteName);
-
-        restTestSuite.setSetupSection(SetupSection.parseIfNext(parser));
-        restTestSuite.setTeardownSection(TeardownSection.parseIfNext(parser));
+        SetupSection setupSection = SetupSection.parseIfNext(parser);
+        TeardownSection teardownSection = TeardownSection.parseIfNext(parser);
 
+        Set<ClientYamlTestSection> testSections = new TreeSet<>();
         while(true) {
             //the "---" section separator is not understood by the yaml parser. null is returned, same as when the parser is closed
             //we need to somehow distinguish between a null in the middle of a test ("---")
@@ -93,27 +97,28 @@ public class ClientYamlTestSuite {
                     break;
                 }
             }
-
             ClientYamlTestSection testSection = ClientYamlTestSection.parse(parser);
-            if (!restTestSuite.addTestSection(testSection)) {
+            if (testSections.add(testSection) == false) {
                 throw new ParsingException(testSection.getLocation(), "duplicate test section [" + testSection.getName() + "]");
             }
         }
 
-        return restTestSuite;
+        return new ClientYamlTestSuite(api, suiteName, setupSection, teardownSection, new ArrayList<>(testSections));
     }
 
     private final String api;
     private final String name;
+    private final SetupSection setupSection;
+    private final TeardownSection teardownSection;
+    private final List<ClientYamlTestSection> testSections;
 
-    private SetupSection setupSection;
-    private TeardownSection teardownSection;
-
-    private Set<ClientYamlTestSection> testSections = new TreeSet<>();
-
-    public ClientYamlTestSuite(String api, String name) {
+    ClientYamlTestSuite(String api, String name, SetupSection setupSection, TeardownSection teardownSection,
+                        List<ClientYamlTestSection> testSections) {
         this.api = api;
         this.name = name;
+        this.setupSection = Objects.requireNonNull(setupSection, "setup section cannot be null");
+        this.teardownSection = Objects.requireNonNull(teardownSection, "teardown section cannot be null");
+        this.testSections = Collections.unmodifiableList(testSections);
     }
 
     public String getApi() {
@@ -132,27 +137,63 @@ public class ClientYamlTestSuite {
         return setupSection;
     }
 
-    public void setSetupSection(SetupSection setupSection) {
-        this.setupSection = setupSection;
-    }
-
     public TeardownSection getTeardownSection() {
         return teardownSection;
     }
 
-    public void setTeardownSection(TeardownSection teardownSection) {
-        this.teardownSection = teardownSection;
+    public void validate() {
+        Stream<String> errors = validateExecutableSections(setupSection.getExecutableSections(), null, setupSection, null);
+        errors = Stream.concat(errors, validateExecutableSections(teardownSection.getDoSections(), null, null, teardownSection));
+        errors = Stream.concat(errors, testSections.stream()
+            .flatMap(section -> validateExecutableSections(section.getExecutableSections(), section, setupSection, teardownSection)));
+        String errorMessage = errors.collect(Collectors.joining(",\n"));
+        if (errorMessage.isEmpty() == false) {
+            throw new IllegalArgumentException(getPath() + ":\n" + errorMessage);
+        }
+    }
+
+    private static Stream<String> validateExecutableSections(List<ExecutableSection> sections,
+                                                   ClientYamlTestSection testSection,
+                                                   SetupSection setupSection, TeardownSection teardownSection) {
+
+        Stream<String> errors = sections.stream().filter(section -> section instanceof DoSection)
+            .map(section -> (DoSection) section)
+            .filter(section -> false == section.getExpectedWarningHeaders().isEmpty())
+            .filter(section -> false == hasSkipFeature("warnings", testSection, setupSection, teardownSection))
+            .map(section -> "attempted to add a [do] with a [warnings] section " +
+                "without a corresponding [\"skip\": \"features\": \"warnings\"] so runners that do not support the [warnings] " +
+                "section can skip the test at line [" + section.getLocation().lineNumber + "]");
+
+        errors = Stream.concat(errors, sections.stream().filter(section -> section instanceof DoSection)
+            .map(section -> (DoSection) section)
+            .filter(section -> NodeSelector.ANY != section.getApiCallSection().getNodeSelector())
+            .filter(section -> false == hasSkipFeature("node_selector", testSection, setupSection, teardownSection))
+            .map(section -> "attempted to add a [do] with a [node_selector] " +
+                "section without a corresponding [\"skip\": \"features\": \"node_selector\"] so runners that do not support the " +
+                "[node_selector] section can skip the test at line [" + section.getLocation().lineNumber + "]"));
+
+        errors = Stream.concat(errors, sections.stream()
+            .filter(section -> section instanceof ContainsAssertion)
+            .filter(section -> false == hasSkipFeature("contains", testSection, setupSection, teardownSection))
+            .map(section -> "attempted to add a [contains] assertion " +
+                "without a corresponding [\"skip\": \"features\": \"contains\"] so runners that do not support the " +
+                "[contains] assertion can skip the test at line [" + section.getLocation().lineNumber + "]"));
+
+        return errors;
+    }
+
+    private static boolean hasSkipFeature(String feature, ClientYamlTestSection testSection,
+                                          SetupSection setupSection, TeardownSection teardownSection) {
+        return (testSection != null && hasSkipFeature(feature, testSection.getSkipSection())) ||
+            (setupSection != null && hasSkipFeature(feature, setupSection.getSkipSection())) ||
+            (teardownSection != null && hasSkipFeature(feature, teardownSection.getSkipSection()));
     }
 
-    /**
-     * Adds a {@link org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection} to the REST suite
-     * @return true if the test section was not already present, false otherwise
-     */
-    public boolean addTestSection(ClientYamlTestSection testSection) {
-        return this.testSections.add(testSection);
+    private static boolean hasSkipFeature(String feature, SkipSection skipSection) {
+        return skipSection != null && skipSection.getFeatures().contains(feature);
     }
 
     public List<ClientYamlTestSection> getTestSections() {
-        return new ArrayList<>(testSections);
+        return testSections;
     }
 }

test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java

@@ -182,7 +182,6 @@ public class DoSection implements ExecutableSection {
         return doSection;
     }
 
-
     private static final Logger logger = LogManager.getLogger(DoSection.class);
 
     private final XContentLocation location;
@@ -206,7 +205,7 @@ public class DoSection implements ExecutableSection {
         return apiCallSection;
     }
 
-    public void setApiCallSection(ApiCallSection apiCallSection) {
+    void setApiCallSection(ApiCallSection apiCallSection) {
         this.apiCallSection = apiCallSection;
     }
 
@@ -214,7 +213,7 @@ public class DoSection implements ExecutableSection {
      * Warning headers that we expect from this response. If the headers don't match exactly this request is considered to have failed.
      * Defaults to emptyList.
      */
-    public List<String> getExpectedWarningHeaders() {
+    List<String> getExpectedWarningHeaders() {
         return expectedWarningHeaders;
     }
 
@@ -222,7 +221,7 @@ public class DoSection implements ExecutableSection {
      * Set the warning headers that we expect from this response. If the headers don't match exactly this request is considered to have
      * failed. Defaults to emptyList.
      */
-    public void setExpectedWarningHeaders(List<String> expectedWarningHeaders) {
+    void setExpectedWarningHeaders(List<String> expectedWarningHeaders) {
         this.expectedWarningHeaders = expectedWarningHeaders;
     }
 

test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetupSection.java

@@ -22,7 +22,9 @@ import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
+import java.util.Objects;
 
 /**
  * Represents a setup section. Holds a skip section and multiple do sections.
@@ -31,7 +33,7 @@ public class SetupSection {
     /**
      * Parse a {@link SetupSection} if the next field is {@code skip}, otherwise returns {@link SetupSection#EMPTY}.
      */
-    public static SetupSection parseIfNext(XContentParser parser) throws IOException {
+    static SetupSection parseIfNext(XContentParser parser) throws IOException {
         ParserUtils.advanceToFieldName(parser);
 
         if ("setup".equals(parser.currentName())) {
@@ -45,58 +47,42 @@ public class SetupSection {
     }
 
     public static SetupSection parse(XContentParser parser) throws IOException {
-        SetupSection setupSection = new SetupSection();
-        setupSection.setSkipSection(SkipSection.parseIfNext(parser));
-
+        SkipSection skipSection = SkipSection.parseIfNext(parser);
+        List<ExecutableSection> executableSections = new ArrayList<>();
         while (parser.currentToken() != XContentParser.Token.END_ARRAY) {
             ParserUtils.advanceToFieldName(parser);
             if ("do".equals(parser.currentName())) {
-                setupSection.addDoSection(DoSection.parse(parser));
+                executableSections.add(DoSection.parse(parser));
             } else if ("set".equals(parser.currentName())) {
-                setupSection.addSetSection(SetSection.parse(parser));
+                executableSections.add(SetSection.parse(parser));
             } else {
                 throw new IllegalArgumentException("section [" + parser.currentName() + "] not supported within setup section");
             }
 
             parser.nextToken();
         }
-
         parser.nextToken();
-
-        return setupSection;
+        return new SetupSection(skipSection, executableSections);
     }
 
-    public static final SetupSection EMPTY;
+    public static final SetupSection EMPTY = new SetupSection(SkipSection.EMPTY, Collections.emptyList());
 
-    static {
-        EMPTY = new SetupSection();
-        EMPTY.setSkipSection(SkipSection.EMPTY);
-    }
-
-    private SkipSection skipSection;
+    private final SkipSection skipSection;
+    private final List<ExecutableSection> executableSections;
 
-    private List<ExecutableSection> executableSections = new ArrayList<>();
+    SetupSection(SkipSection skipSection, List<ExecutableSection> executableSections) {
+        this.skipSection = Objects.requireNonNull(skipSection, "skip section cannot be null");
+        this.executableSections = Collections.unmodifiableList(executableSections);
+    }
 
     public SkipSection getSkipSection() {
         return skipSection;
     }
 
-    public void setSkipSection(SkipSection skipSection) {
-        this.skipSection = skipSection;
-    }
-
     public List<ExecutableSection> getExecutableSections() {
         return executableSections;
     }
 
-    public void addDoSection(DoSection doSection) {
-        this.executableSections.add(doSection);
-    }
-
-    public void addSetSection(SetSection setSection) {
-        this.executableSections.add(setSection);
-    }
-
     public boolean isEmpty() {
         return EMPTY.equals(this);
     }

test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/TeardownSection.java

@@ -24,13 +24,15 @@ import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
+import java.util.Objects;
 
 public class TeardownSection {
     /**
      * Parse a {@link TeardownSection} if the next field is {@code skip}, otherwise returns {@link TeardownSection#EMPTY}.
      */
-    public static TeardownSection parseIfNext(XContentParser parser) throws IOException {
+    static TeardownSection parseIfNext(XContentParser parser) throws IOException {
         ParserUtils.advanceToFieldName(parser);
 
         if ("teardown".equals(parser.currentName())) {
@@ -44,50 +46,40 @@ public class TeardownSection {
     }
 
     public static TeardownSection parse(XContentParser parser) throws IOException {
-        TeardownSection teardownSection = new TeardownSection();
-        teardownSection.setSkipSection(SkipSection.parseIfNext(parser));
-
+        SkipSection skipSection = SkipSection.parseIfNext(parser);
+        List<ExecutableSection> executableSections = new ArrayList<>();
         while (parser.currentToken() != XContentParser.Token.END_ARRAY) {
             ParserUtils.advanceToFieldName(parser);
             if (!"do".equals(parser.currentName())) {
                 throw new ParsingException(parser.getTokenLocation(),
                         "section [" + parser.currentName() + "] not supported within teardown section");
             }
-
-            teardownSection.addDoSection(DoSection.parse(parser));
+            executableSections.add(DoSection.parse(parser));
             parser.nextToken();
         }
 
         parser.nextToken();
-        return teardownSection;
+        return new TeardownSection(skipSection, executableSections);
     }
 
-    public static final TeardownSection EMPTY;
+    public static final TeardownSection EMPTY = new TeardownSection(SkipSection.EMPTY, Collections.emptyList());
 
-    static {
-        EMPTY = new TeardownSection();
-        EMPTY.setSkipSection(SkipSection.EMPTY);
-    }
+    private final SkipSection skipSection;
+    private final List<ExecutableSection> doSections;
 
-    private SkipSection skipSection;
-    private List<DoSection> doSections = new ArrayList<>();
+    TeardownSection(SkipSection skipSection, List<ExecutableSection> doSections) {
+        this.skipSection = Objects.requireNonNull(skipSection, "skip section cannot be null");
+        this.doSections = Collections.unmodifiableList(doSections);
+    }
 
     public SkipSection getSkipSection() {
         return skipSection;
     }
 
-    public void setSkipSection(SkipSection skipSection) {
-        this.skipSection = skipSection;
-    }
-
-    public List<DoSection> getDoSections() {
+    public List<ExecutableSection> getDoSections() {
         return doSections;
     }
 
-    public void addDoSection(DoSection doSection) {
-        this.doSections.add(doSection);
-    }
-
     public boolean isEmpty() {
         return EMPTY.equals(this);
     }

test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSectionTests.java

@@ -20,78 +20,19 @@
 package org.elasticsearch.test.rest.yaml.section;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.client.NodeSelector;
 import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.xcontent.XContentLocation;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.yaml.YamlXContent;
 
 import java.io.IOException;
 import java.util.Map;
 
-import static java.util.Collections.singletonList;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
 public class ClientYamlTestSectionTests extends AbstractClientYamlTestFragmentParserTestCase {
-    public void testAddingDoWithoutSkips() {
-        int lineNumber = between(1, 10000);
-        ClientYamlTestSection section = new ClientYamlTestSection(new XContentLocation(0, 0), "test");
-        section.setSkipSection(SkipSection.EMPTY);
-        DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
-        doSection.setApiCallSection(new ApiCallSection("test"));
-        section.addExecutableSection(doSection);
-    }
-
-    public void testAddingDoWithWarningWithSkip() {
-        int lineNumber = between(1, 10000);
-        ClientYamlTestSection section = new ClientYamlTestSection(new XContentLocation(0, 0), "test");
-        section.setSkipSection(new SkipSection(null, singletonList("warnings"), null));
-        DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
-        doSection.setExpectedWarningHeaders(singletonList("foo"));
-        doSection.setApiCallSection(new ApiCallSection("test"));
-        section.addExecutableSection(doSection);
-    }
-
-    public void testAddingDoWithWarningWithSkipButNotWarnings() {
-        int lineNumber = between(1, 10000);
-        ClientYamlTestSection section = new ClientYamlTestSection(new XContentLocation(0, 0), "test");
-        section.setSkipSection(new SkipSection(null, singletonList("yaml"), null));
-        DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
-        doSection.setExpectedWarningHeaders(singletonList("foo"));
-        doSection.setApiCallSection(new ApiCallSection("test"));
-        Exception e = expectThrows(IllegalArgumentException.class, () -> section.addExecutableSection(doSection));
-        assertEquals("Attempted to add a [do] with a [warnings] section without a corresponding [skip] so runners that do not support the"
-                + " [warnings] section can skip the test at line [" + lineNumber + "]", e.getMessage());
-    }
-
-    public void testAddingDoWithNodeSelectorWithSkip() {
-        int lineNumber = between(1, 10000);
-        ClientYamlTestSection section = new ClientYamlTestSection(new XContentLocation(0, 0), "test");
-        section.setSkipSection(new SkipSection(null, singletonList("node_selector"), null));
-        DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
-        ApiCallSection apiCall = new ApiCallSection("test");
-        apiCall.setNodeSelector(NodeSelector.SKIP_DEDICATED_MASTERS);
-        doSection.setApiCallSection(apiCall);
-        section.addExecutableSection(doSection);
-    }
-
-    public void testAddingDoWithNodeSelectorWithSkipButNotWarnings() {
-        int lineNumber = between(1, 10000);
-        ClientYamlTestSection section = new ClientYamlTestSection(new XContentLocation(0, 0), "test");
-        section.setSkipSection(new SkipSection(null, singletonList("yaml"), null));
-        DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
-        ApiCallSection apiCall = new ApiCallSection("test");
-        apiCall.setNodeSelector(NodeSelector.SKIP_DEDICATED_MASTERS);
-        doSection.setApiCallSection(apiCall);
-        Exception e = expectThrows(IllegalArgumentException.class, () -> section.addExecutableSection(doSection));
-        assertEquals("Attempted to add a [do] with a [node_selector] section without a corresponding"
-                + " [skip] so runners that do not support the [node_selector] section can skip the test at"
-                + " line [" + lineNumber + "]", e.getMessage());
-    }
-
     public void testWrongIndentation() throws Exception {
         {
             XContentParser parser = createParser(YamlXContent.yamlXContent,
@@ -297,7 +238,7 @@ public class ClientYamlTestSectionTests extends AbstractClientYamlTestFragmentPa
         LengthAssertion lengthAssertion = (LengthAssertion) testSection.getExecutableSections().get(6);
         assertThat(lengthAssertion.getField(), equalTo("_index"));
         assertThat(lengthAssertion.getExpectedValue(), instanceOf(Integer.class));
-        assertThat((Integer) lengthAssertion.getExpectedValue(), equalTo(6));
+        assertThat(lengthAssertion.getExpectedValue(), equalTo(6));
 
         IsFalseAssertion falseAssertion = (IsFalseAssertion)testSection.getExecutableSections().get(7);
         assertThat(falseAssertion.getField(), equalTo("whatever"));
@@ -305,12 +246,12 @@ public class ClientYamlTestSectionTests extends AbstractClientYamlTestFragmentPa
         GreaterThanAssertion greaterThanAssertion = (GreaterThanAssertion) testSection.getExecutableSections().get(8);
         assertThat(greaterThanAssertion.getField(), equalTo("size"));
         assertThat(greaterThanAssertion.getExpectedValue(), instanceOf(Integer.class));
-        assertThat((Integer) greaterThanAssertion.getExpectedValue(), equalTo(5));
+        assertThat(greaterThanAssertion.getExpectedValue(), equalTo(5));
 
         LessThanAssertion lessThanAssertion = (LessThanAssertion) testSection.getExecutableSections().get(9);
         assertThat(lessThanAssertion.getField(), equalTo("size"));
         assertThat(lessThanAssertion.getExpectedValue(), instanceOf(Integer.class));
-        assertThat((Integer) lessThanAssertion.getExpectedValue(), equalTo(10));
+        assertThat(lessThanAssertion.getExpectedValue(), equalTo(10));
     }
 
     public void testSmallSection() throws Exception {
@@ -327,28 +268,4 @@ public class ClientYamlTestSectionTests extends AbstractClientYamlTestFragmentPa
         assertThat(testSection.getName(), equalTo("node_info test"));
         assertThat(testSection.getExecutableSections().size(), equalTo(3));
     }
-
-    public void testAddingContainsWithoutSkip() {
-        int lineNumber = between(1, 10000);
-        ClientYamlTestSection section = new ClientYamlTestSection(new XContentLocation(0, 0), "test");
-        if (randomBoolean()) {
-            section.setSkipSection(new SkipSection(null, singletonList("yaml"), null));
-        } else {
-            section.setSkipSection(SkipSection.EMPTY);
-        }
-        Exception e = expectThrows(
-            IllegalArgumentException.class,
-            () -> section.addExecutableSection(
-                new ContainsAssertion(
-                    new XContentLocation(lineNumber, 0),
-                    randomAlphaOfLength(randomIntBetween(3, 30)),
-                    randomDouble()
-                )
-            )
-        );
-        assertEquals("Attempted to add a [contains] assertion without a corresponding " +
-            "[skip: \"features\": \"contains\"] so runners that do not support the [contains] assertion " +
-            "can skip the test at line [" + lineNumber + "]", e.getMessage());
-    }
-
 }

test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuiteTests.java

@@ -20,11 +20,17 @@
 package org.elasticsearch.test.rest.yaml.section;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.client.NodeSelector;
 import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.xcontent.XContentLocation;
 import org.elasticsearch.common.xcontent.yaml.YamlXContent;
 
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 
+import static java.util.Collections.singletonList;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
@@ -106,9 +112,11 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars
             assertThat(restTestSuite.getTeardownSection().isEmpty(), equalTo(false));
             assertThat(restTestSuite.getTeardownSection().getSkipSection().isEmpty(), equalTo(true));
             assertThat(restTestSuite.getTeardownSection().getDoSections().size(), equalTo(1));
-            assertThat(restTestSuite.getTeardownSection().getDoSections().get(0).getApiCallSection().getApi(), equalTo("indices.delete"));
-            assertThat(restTestSuite.getTeardownSection().getDoSections().get(0).getApiCallSection().getParams().size(), equalTo(1));
-            assertThat(restTestSuite.getTeardownSection().getDoSections().get(0).getApiCallSection().getParams().get("index"),
+            assertThat(((DoSection)restTestSuite.getTeardownSection().getDoSections().get(0)).getApiCallSection().getApi(),
+                equalTo("indices.delete"));
+            assertThat(((DoSection)restTestSuite.getTeardownSection().getDoSections().get(0)).getApiCallSection().getParams().size(),
+                equalTo(1));
+            assertThat(((DoSection)restTestSuite.getTeardownSection().getDoSections().get(0)).getApiCallSection().getParams().get("index"),
                 equalTo("test_index"));
         } else {
             assertThat(restTestSuite.getTeardownSection().isEmpty(), equalTo(true));
@@ -378,4 +386,198 @@ public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentPars
             ClientYamlTestSuite.parse(getTestClass().getName(), getTestName(), parser));
         assertThat(e.getMessage(), containsString("duplicate test section"));
     }
+
+    public void testAddingDoWithoutSkips() {
+        int lineNumber = between(1, 10000);
+        DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
+        doSection.setApiCallSection(new ApiCallSection("test"));
+        ClientYamlTestSection section = new ClientYamlTestSection(new XContentLocation(0, 0), "test",
+            SkipSection.EMPTY, Collections.singletonList(doSection));
+        ClientYamlTestSuite clientYamlTestSuite = new ClientYamlTestSuite("api", "name", SetupSection.EMPTY, TeardownSection.EMPTY,
+            Collections.singletonList(section));
+        clientYamlTestSuite.validate();
+    }
+
+    public void testAddingDoWithWarningWithoutSkipWarnings() {
+        int lineNumber = between(1, 10000);
+        DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
+        doSection.setExpectedWarningHeaders(singletonList("foo"));
+        doSection.setApiCallSection(new ApiCallSection("test"));
+        ClientYamlTestSuite testSuite = createTestSuite(doSection);
+        Exception e = expectThrows(IllegalArgumentException.class, testSuite::validate);
+        assertEquals("api/name:\nattempted to add a [do] with a [warnings] section without a corresponding " +
+            "[\"skip\": \"features\": \"warnings\"] so runners that do not support the [warnings] section can skip the test " +
+            "at line [" + lineNumber + "]", e.getMessage());
+    }
+
+    public void testAddingDoWithNodeSelectorWithoutSkipNodeSelector() {
+        int lineNumber = between(1, 10000);
+        DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
+        ApiCallSection apiCall = new ApiCallSection("test");
+        apiCall.setNodeSelector(NodeSelector.SKIP_DEDICATED_MASTERS);
+        doSection.setApiCallSection(apiCall);
+        ClientYamlTestSuite testSuite = createTestSuite(doSection);
+        Exception e = expectThrows(IllegalArgumentException.class, testSuite::validate);
+        assertEquals("api/name:\nattempted to add a [do] with a [node_selector] section without a corresponding"
+            + " [\"skip\": \"features\": \"node_selector\"] so runners that do not support the [node_selector] section can skip the test at"
+            + " line [" + lineNumber + "]", e.getMessage());
+    }
+
+    public void testAddingContainsWithoutSkipContains() {
+        int lineNumber = between(1, 10000);
+        ContainsAssertion containsAssertion = new ContainsAssertion(
+            new XContentLocation(lineNumber, 0),
+            randomAlphaOfLength(randomIntBetween(3, 30)),
+            randomDouble());
+        ClientYamlTestSuite testSuite = createTestSuite(containsAssertion);
+        Exception e = expectThrows(IllegalArgumentException.class, testSuite::validate);
+        assertEquals("api/name:\nattempted to add a [contains] assertion without a corresponding " +
+            "[\"skip\": \"features\": \"contains\"] so runners that do not support the [contains] assertion " +
+            "can skip the test at line [" + lineNumber + "]", e.getMessage());
+    }
+
+    public void testMultipleValidationErrors() {
+        int firstLineNumber = between(1, 10000);
+        List<ClientYamlTestSection> sections = new ArrayList<>();
+        {
+            ContainsAssertion containsAssertion = new ContainsAssertion(
+                new XContentLocation(firstLineNumber, 0),
+                randomAlphaOfLength(randomIntBetween(3, 30)),
+                randomDouble());
+            sections.add(new ClientYamlTestSection(
+                new XContentLocation(0, 0), "section1", SkipSection.EMPTY, Collections.singletonList(containsAssertion)));
+        }
+        int secondLineNumber = between(1, 10000);
+        int thirdLineNumber = between(1, 10000);
+        List<ExecutableSection> doSections = new ArrayList<>();
+        {
+            DoSection doSection = new DoSection(new XContentLocation(secondLineNumber, 0));
+            doSection.setExpectedWarningHeaders(singletonList("foo"));
+            doSection.setApiCallSection(new ApiCallSection("test"));
+            doSections.add(doSection);
+        }
+        {
+            DoSection doSection = new DoSection(new XContentLocation(thirdLineNumber, 0));
+            ApiCallSection apiCall = new ApiCallSection("test");
+            apiCall.setNodeSelector(NodeSelector.SKIP_DEDICATED_MASTERS);
+            doSection.setApiCallSection(apiCall);
+            doSections.add(doSection);
+        }
+        sections.add(new ClientYamlTestSection(new XContentLocation(0, 0), "section2", SkipSection.EMPTY, doSections));
+
+        ClientYamlTestSuite testSuite = new ClientYamlTestSuite("api", "name", SetupSection.EMPTY, TeardownSection.EMPTY, sections);
+        Exception e = expectThrows(IllegalArgumentException.class, testSuite::validate);
+        assertEquals("api/name:\n" +
+            "attempted to add a [contains] assertion without a corresponding [\"skip\": \"features\": \"contains\"] so runners " +
+                "that do not support the [contains] assertion can skip the test at line [" + firstLineNumber + "],\n" +
+            "attempted to add a [do] with a [warnings] section without a corresponding [\"skip\": \"features\": \"warnings\"] so " +
+                "runners that do not support the [warnings] section can skip the test at line [" + secondLineNumber + "],\n" +
+            "attempted to add a [do] with a [node_selector] section without a corresponding [\"skip\": \"features\": \"node_selector\"] " +
+                "so runners that do not support the [node_selector] section can skip the test at line [" + thirdLineNumber + "]",
+            e.getMessage());
+    }
+
+    private static ClientYamlTestSuite createTestSuite(ExecutableSection executableSection) {
+        final SetupSection setupSection;
+        final TeardownSection teardownSection;
+        final ClientYamlTestSection clientYamlTestSection;
+        switch(randomIntBetween(0, 2)) {
+            case 0:
+                setupSection = new SetupSection(SkipSection.EMPTY, Collections.singletonList(executableSection));
+                teardownSection = TeardownSection.EMPTY;
+                clientYamlTestSection = new ClientYamlTestSection(new XContentLocation(0, 0), "test",
+                    SkipSection.EMPTY, Collections.emptyList());
+                break;
+            case 1:
+                setupSection = SetupSection.EMPTY;
+                teardownSection = new TeardownSection(SkipSection.EMPTY, Collections.singletonList(executableSection));
+                clientYamlTestSection = new ClientYamlTestSection(new XContentLocation(0, 0), "test",
+                    SkipSection.EMPTY, Collections.emptyList());
+                break;
+            case 2:
+                setupSection = SetupSection.EMPTY;
+                teardownSection = TeardownSection.EMPTY;
+                clientYamlTestSection = new ClientYamlTestSection(new XContentLocation(0, 0), "test",
+                    SkipSection.EMPTY, Collections.singletonList(executableSection));
+                break;
+            default:
+                throw new UnsupportedOperationException();
+        }
+
+        return new ClientYamlTestSuite("api", "name", setupSection, teardownSection,
+            Collections.singletonList(clientYamlTestSection));
+    }
+
+    public void testAddingDoWithWarningWithSkip() {
+        int lineNumber = between(1, 10000);
+        DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
+        doSection.setExpectedWarningHeaders(singletonList("foo"));
+        doSection.setApiCallSection(new ApiCallSection("test"));
+        SkipSection skipSection = new SkipSection(null, singletonList("warnings"), null);
+        createTestSuiteAndValidate(skipSection, doSection);
+    }
+
+    public void testAddingDoWithNodeSelectorWithSkip() {
+        int lineNumber = between(1, 10000);
+        SkipSection skipSection = new SkipSection(null, singletonList("node_selector"), null);
+        DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
+        ApiCallSection apiCall = new ApiCallSection("test");
+        apiCall.setNodeSelector(NodeSelector.SKIP_DEDICATED_MASTERS);
+        doSection.setApiCallSection(apiCall);
+        createTestSuiteAndValidate(skipSection, doSection);
+    }
+
+    public void testAddingContainsWithSkip() {
+        int lineNumber = between(1, 10000);
+        SkipSection skipSection = new SkipSection(null, singletonList("contains"), null);
+        ContainsAssertion containsAssertion = new ContainsAssertion(
+            new XContentLocation(lineNumber, 0),
+            randomAlphaOfLength(randomIntBetween(3, 30)),
+            randomDouble());
+        createTestSuiteAndValidate(skipSection, containsAssertion);
+    }
+
+    private static void createTestSuiteAndValidate(SkipSection skipSection, ExecutableSection executableSection) {
+        final SetupSection setupSection;
+        final TeardownSection teardownSection;
+        final ClientYamlTestSection clientYamlTestSection;
+        switch(randomIntBetween(0, 4)) {
+            case 0:
+                setupSection = new SetupSection(skipSection, Collections.emptyList());
+                teardownSection = TeardownSection.EMPTY;
+                clientYamlTestSection = new ClientYamlTestSection(new XContentLocation(0, 0), "test",
+                    SkipSection.EMPTY, Collections.singletonList(executableSection));
+                break;
+            case 1:
+                setupSection = SetupSection.EMPTY;
+                teardownSection = new TeardownSection(skipSection, Collections.emptyList());
+                clientYamlTestSection = new ClientYamlTestSection(new XContentLocation(0, 0), "test",
+                    SkipSection.EMPTY, Collections.singletonList(executableSection));
+                break;
+            case 2:
+                setupSection = SetupSection.EMPTY;
+                teardownSection = TeardownSection.EMPTY;
+                clientYamlTestSection = new ClientYamlTestSection(new XContentLocation(0, 0), "test",
+                    skipSection, Collections.singletonList(executableSection));
+                break;
+            case 3:
+                setupSection = new SetupSection(skipSection, Collections.singletonList(executableSection));
+                teardownSection = TeardownSection.EMPTY;
+                clientYamlTestSection = new ClientYamlTestSection(new XContentLocation(0, 0), "test",
+                    SkipSection.EMPTY, randomBoolean() ? Collections.emptyList() : Collections.singletonList(executableSection));
+                break;
+            case 4:
+                setupSection = SetupSection.EMPTY;
+                teardownSection = new TeardownSection(skipSection, Collections.singletonList(executableSection));
+                clientYamlTestSection = new ClientYamlTestSection(new XContentLocation(0, 0), "test",
+                    SkipSection.EMPTY, randomBoolean() ? Collections.emptyList() : Collections.singletonList(executableSection));
+
+                break;
+            default:
+                throw new UnsupportedOperationException();
+        }
+        ClientYamlTestSuite clientYamlTestSuite = new ClientYamlTestSuite("api", "name", setupSection, teardownSection,
+            Collections.singletonList(clientYamlTestSection));
+        clientYamlTestSuite.validate();
+    }
 }

test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/TeardownSectionTests.java

@@ -49,8 +49,8 @@ public class TeardownSectionTests extends AbstractClientYamlTestFragmentParserTe
         assertThat(section, notNullValue());
         assertThat(section.getSkipSection().isEmpty(), equalTo(true));
         assertThat(section.getDoSections().size(), equalTo(2));
-        assertThat(section.getDoSections().get(0).getApiCallSection().getApi(), equalTo("delete"));
-        assertThat(section.getDoSections().get(1).getApiCallSection().getApi(), equalTo("delete2"));
+        assertThat(((DoSection)section.getDoSections().get(0)).getApiCallSection().getApi(), equalTo("delete"));
+        assertThat(((DoSection)section.getDoSections().get(1)).getApiCallSection().getApi(), equalTo("delete2"));
     }
 
     public void testParseWithSkip() throws Exception {
@@ -79,7 +79,7 @@ public class TeardownSectionTests extends AbstractClientYamlTestFragmentParserTe
         assertThat(section.getSkipSection().getUpperVersion(), equalTo(Version.V_6_3_0));
         assertThat(section.getSkipSection().getReason(), equalTo("there is a reason"));
         assertThat(section.getDoSections().size(), equalTo(2));
-        assertThat(section.getDoSections().get(0).getApiCallSection().getApi(), equalTo("delete"));
-        assertThat(section.getDoSections().get(1).getApiCallSection().getApi(), equalTo("delete2"));
+        assertThat(((DoSection)section.getDoSections().get(0)).getApiCallSection().getApi(), equalTo("delete"));
+        assertThat(((DoSection)section.getDoSections().get(1)).getApiCallSection().getApi(), equalTo("delete2"));
     }
 }