[Compatible REST Testing] Introduce skip test via transformation (#78398)

Prior to this change, the only way to express that a compatible REST test
should be skipped was via the blacklist of the ES test runner. While this
works for ES, it requires any consumers of the compatible REST tests copy/paste
the list of tests that should not be executed.

This commit introduces 2 new transforms for the compatible REST tests.
skipTest - adds a skip section to the named test with a given reason
skipTestsByFilePattern - add a skip section to the global setup for
any test file that matches the file pattern
and includes the given reason.

All uses of the blacklist have been replaced by the new skip transforms.

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/YamlRestCompatTestPluginFuncTest.groovy

@@ -196,6 +196,7 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest {
                yamlRestTestImplementation "junit:junit:4.12"
             }
             tasks.named("yamlRestTestV${compatibleVersion}CompatTransform").configure({ task ->
+              task.skipTest("test/test/two", "This is a test to skip test two")
               task.replaceValueInMatch("_type", "_doc")
               task.replaceValueInMatch("_source.values", ["z", "x", "y"], "one")
               task.removeMatch("_source.blah")
@@ -333,6 +334,9 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest {
 
         ---
         two:
+        - skip:
+            version: "all"
+            reason: "This is a test to skip test two"
         - do:
             get:
               index: "test2"

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java

@@ -18,6 +18,8 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.fasterxml.jackson.databind.node.TextNode;
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
 import com.fasterxml.jackson.dataformat.yaml.YAMLParser;
+
+import org.apache.commons.lang3.tuple.Pair;
 import org.elasticsearch.gradle.Version;
 import org.elasticsearch.gradle.VersionProperties;
 import org.elasticsearch.gradle.internal.test.rest.transform.RestTestTransform;
@@ -30,6 +32,7 @@ import org.elasticsearch.gradle.internal.test.rest.transform.match.AddMatch;
 import org.elasticsearch.gradle.internal.test.rest.transform.match.RemoveMatch;
 import org.elasticsearch.gradle.internal.test.rest.transform.match.ReplaceKeyInMatch;
 import org.elasticsearch.gradle.internal.test.rest.transform.match.ReplaceValueInMatch;
+import org.elasticsearch.gradle.internal.test.rest.transform.skip.Skip;
 import org.elasticsearch.gradle.internal.test.rest.transform.text.ReplaceIsFalse;
 import org.elasticsearch.gradle.internal.test.rest.transform.text.ReplaceIsTrue;
 import org.elasticsearch.gradle.internal.test.rest.transform.text.ReplaceTextual;
@@ -41,6 +44,7 @@ import org.gradle.api.file.DirectoryProperty;
 import org.gradle.api.file.FileSystemOperations;
 import org.gradle.api.file.FileTree;
 import org.gradle.api.model.ObjectFactory;
+import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.InputFiles;
 import org.gradle.api.tasks.Internal;
 import org.gradle.api.tasks.Nested;
@@ -51,17 +55,20 @@ import org.gradle.api.tasks.util.PatternFilterable;
 import org.gradle.api.tasks.util.PatternSet;
 import org.gradle.internal.Factory;
 
-import javax.inject.Inject;
 import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
+import javax.inject.Inject;
 
 /**
  * A task to transform REST tests for use in REST API compatibility before they are executed.
@@ -81,7 +88,12 @@ public class RestCompatTestTransformTask extends DefaultTask {
     private final DirectoryProperty sourceDirectory;
     private final DirectoryProperty outputDirectory;
     private final PatternFilterable testPatternSet;
+    private final Factory<PatternSet> patternSetFactory;
     private final List<RestTestTransform<?>> transformations = new ArrayList<>();
+    // PatternFilterable -> reason why skipped.
+    private final Map<PatternFilterable, String> skippedTestByFilePatternTransformations = new HashMap<>();
+    // PatternFilterable -> list of full test names and reasons. Needed for 1 pattern may include many tests and reasons
+    private final Map<PatternFilterable, List<Pair<String, String>>> skippedTestByTestNameTransformations = new HashMap<>();
 
     @Inject
     public RestCompatTestTransformTask(
@@ -89,6 +101,7 @@ public class RestCompatTestTransformTask extends DefaultTask {
         Factory<PatternSet> patternSetFactory,
         ObjectFactory objectFactory
     ) {
+        this.patternSetFactory = patternSetFactory;
         this.fileSystemOperations = fileSystemOperations;
         this.compatibleVersion = Version.fromString(VersionProperties.getVersions().get("elasticsearch")).getMajor() - 1;
         this.sourceDirectory = objectFactory.directoryProperty();
@@ -112,6 +125,37 @@ public class RestCompatTestTransformTask extends DefaultTask {
         return true;
     }
 
+    public void skipTest(String fullTestName, String reason) {
+        //The tests are defined by 3 parts a/b/c where
+        // a = the folder name
+        // b = the file name without the .yml extension
+        // c = the test name inside the .yml
+        // For example: indices.get_mapping/20_missing_type/Non-existent type returns 404
+        // However, the folder can be arbitrarily nest so, a == a1/a2/a3, and the test name can include forward slashes, so c == c1/c2/c3
+        // So we also need to support a1/a2/a3/b/c1/c2/c3
+
+        String[] testParts = fullTestName.split("/");
+        if(testParts.length < 3 ){
+            throw new IllegalArgumentException("To skip tests, all 3 parts [folder/file/test name] must be defined. found [" + fullTestName + "]");
+        }
+
+        PatternSet skippedPatternSet = patternSetFactory.create();
+        //create file patterns for all a1/a2/a3/b.yml possibilities.
+        for(int i = testParts.length - 1; i > 1; i-- ){
+            final String lastPart = testParts[i];
+            String filePattern = "**/" + Arrays.stream(testParts).takeWhile(x -> x.equals(lastPart) == false).collect(Collectors.joining("/")) + ".yml";
+            skippedPatternSet.include(filePattern);
+        }
+
+        skippedTestByTestNameTransformations.computeIfAbsent(skippedPatternSet, k -> new ArrayList<>()).add(Pair.of(fullTestName, reason));
+    }
+
+    public void skipTestsByFilePattern(String filePattern, String reason) {
+        PatternSet skippedPatternSet = patternSetFactory.create();
+        skippedPatternSet.include(filePattern);
+        skippedTestByFilePatternTransformations.put(skippedPatternSet, reason);
+    }
+
     /**
      * Replaces all the values of a match assertion for all project REST tests.
      * For example "match":{"_type": "foo"} to "match":{"_type": "bar"}
@@ -136,10 +180,11 @@ public class RestCompatTestTransformTask extends DefaultTask {
 
     /**
      * A transformation to replace the key in a do section.
+     *
+     * @param oldKeyName the key name directly under do to replace.
+     * @param newKeyName the new key name directly under do.
+     * @param testName   the testName to apply replacement
      * @see ReplaceKeyInDo
-     * @param oldKeyName   the key name directly under do to replace.
-     * @param newKeyName   the new key name directly under do.
-     * @param testName the testName to apply replacement
      */
     public void replaceKeyInDo(String oldKeyName, String newKeyName, String testName) {
         transformations.add(new ReplaceKeyInDo(oldKeyName, newKeyName, testName));
@@ -147,9 +192,10 @@ public class RestCompatTestTransformTask extends DefaultTask {
 
     /**
      * A transformation to replace the key in a do section for given REST test.
+     *
+     * @param oldKeyName the key name directly under do to replace.
+     * @param newKeyName the new key name directly under do.
      * @see ReplaceKeyInDo
-     * @param oldKeyName   the key name directly under do to replace.
-     * @param newKeyName   the new key name directly under do.
      */
     public void replaceKeyInDo(String oldKeyName, String newKeyName) {
         transformations.add(new ReplaceKeyInDo(oldKeyName, newKeyName, null));
@@ -157,9 +203,10 @@ public class RestCompatTestTransformTask extends DefaultTask {
 
     /**
      * A transformation to replace the key in a length assertion.
+     *
+     * @param oldKeyName the key name directly under length to replace.
+     * @param newKeyName the new key name directly under length.
      * @see ReplaceKeyInLength
-     * @param oldKeyName   the key name directly under length to replace.
-     * @param newKeyName   the new key name directly under length.
      */
     public void replaceKeyInLength(String oldKeyName, String newKeyName) {
         transformations.add(new ReplaceKeyInLength(oldKeyName, newKeyName, null));
@@ -179,6 +226,7 @@ public class RestCompatTestTransformTask extends DefaultTask {
     /**
      * Replaces all the values of a length assertion for the given REST test.
      * For example "length":{"x": 1} to "length":{"x": 99}
+     *
      * @param subKey   the key name directly under match to replace. For example "x"
      * @param value    the value used in the replacement. For example 99
      * @param testName the testName to apply replacement
@@ -189,9 +237,10 @@ public class RestCompatTestTransformTask extends DefaultTask {
 
     /**
      * A transformation to replace the key in a match assertion.
+     *
+     * @param oldKeyName the key name directly under match to replace.
+     * @param newKeyName the new key name directly under match.
      * @see ReplaceKeyInMatch
-     * @param oldKeyName   the key name directly under match to replace.
-     * @param newKeyName   the new key name directly under match.
      */
     public void replaceKeyInMatch(String oldKeyName, String newKeyName) {
         transformations.add(new ReplaceKeyInMatch(oldKeyName, newKeyName, null));
@@ -202,7 +251,7 @@ public class RestCompatTestTransformTask extends DefaultTask {
      * For example "is_true": "value_to_replace" to "is_true": "value_replaced"
      *
      * @param oldValue the value that has to match and will be replaced
-     * @param newValue  the value used in the replacement
+     * @param newValue the value used in the replacement
      */
     public void replaceIsTrue(String oldValue, Object newValue) {
         transformations.add(new ReplaceIsTrue(oldValue, MAPPER.convertValue(newValue, TextNode.class)));
@@ -213,7 +262,7 @@ public class RestCompatTestTransformTask extends DefaultTask {
      * For example "is_false": "value_to_replace" to "is_false": "value_replaced"
      *
      * @param oldValue the value that has to match and will be replaced
-     * @param newValue  the value used in the replacement
+     * @param newValue the value used in the replacement
      */
     public void replaceIsFalse(String oldValue, Object newValue) {
         transformations.add(new ReplaceIsFalse(oldValue, MAPPER.convertValue(newValue, TextNode.class)));
@@ -224,8 +273,8 @@ public class RestCompatTestTransformTask extends DefaultTask {
      * For example "is_false": "value_to_replace" to "is_false": "value_replaced"
      *
      * @param oldValue the value that has to match and will be replaced
-     * @param newValue  the value used in the replacement
-      @param testName the testName to apply replacement
+     * @param newValue the value used in the replacement
+     * @param testName the testName to apply replacement
      */
     public void replaceIsFalse(String oldValue, Object newValue, String testName) {
         transformations.add(new ReplaceIsFalse(oldValue, MAPPER.convertValue(newValue, TextNode.class), testName));
@@ -235,9 +284,9 @@ public class RestCompatTestTransformTask extends DefaultTask {
      * Replaces all the values of a given key/value pairs for all project REST tests.
      * For example "foo": "bar" can replaced as "foo": "baz"
      *
-     * @param key the key to find
+     * @param key      the key to find
      * @param oldValue the value of that key to find
-     * @param newValue  the value used in the replacement
+     * @param newValue the value used in the replacement
      */
     public void replaceValueTextByKeyValue(String key, String oldValue, Object newValue) {
         transformations.add(new ReplaceTextual(key, oldValue, MAPPER.convertValue(newValue, TextNode.class)));
@@ -247,9 +296,9 @@ public class RestCompatTestTransformTask extends DefaultTask {
      * Replaces all the values of a given key/value pairs for given REST test.
      * For example "foo": "bar" can replaced as "foo": "baz"
      *
-     * @param key the key to find
+     * @param key      the key to find
      * @param oldValue the value of that key to find
-     * @param newValue  the value used in the replacement
+     * @param newValue the value used in the replacement
      * @param testName the testName to apply replacement
      */
     public void replaceValueTextByKeyValue(String key, String oldValue, Object newValue, String testName) {
@@ -293,6 +342,7 @@ public class RestCompatTestTransformTask extends DefaultTask {
 
     /**
      * Adds one or more warnings to the given test
+     *
      * @param testName the test name to add the warning
      * @param warnings the warning(s) to add
      */
@@ -302,7 +352,8 @@ public class RestCompatTestTransformTask extends DefaultTask {
 
     /**
      * Adds one or more regex warnings to the given test
-     * @param testName the test name to add the regex warning
+     *
+     * @param testName      the test name to add the regex warning
      * @param warningsRegex the regex warning(s) to add
      */
     public void addWarningRegex(String testName, String... warningsRegex) {
@@ -311,6 +362,7 @@ public class RestCompatTestTransformTask extends DefaultTask {
 
     /**
      * Removes one or more warnings
+     *
      * @param warnings the warning(s) to remove
      */
     public void removeWarning(String... warnings) {
@@ -319,6 +371,7 @@ public class RestCompatTestTransformTask extends DefaultTask {
 
     /**
      * Removes one or more warnings
+     *
      * @param warnings the warning(s) to remove
      * @param testName the test name to remove the warning
      */
@@ -328,6 +381,7 @@ public class RestCompatTestTransformTask extends DefaultTask {
 
     /**
      * Adds one or more allowed warnings
+     *
      * @param allowedWarnings the warning(s) to add
      */
     public void addAllowedWarning(String... allowedWarnings) {
@@ -336,6 +390,7 @@ public class RestCompatTestTransformTask extends DefaultTask {
 
     /**
      * Adds one or more allowed regular expression warnings
+     *
      * @param allowedWarningsRegex the regex warning(s) to add
      */
     public void addAllowedWarningRegex(String... allowedWarningsRegex) {
@@ -344,6 +399,7 @@ public class RestCompatTestTransformTask extends DefaultTask {
 
     /**
      * Adds one or more allowed regular expression warnings
+     *
      * @param allowedWarningsRegex the regex warning(s) to add
      * @testName the test name to add a allowedWarningRegex
      */
@@ -367,12 +423,44 @@ public class RestCompatTestTransformTask extends DefaultTask {
         // clean the output directory to ensure no stale files persist
         fileSystemOperations.delete(d -> d.delete(outputDirectory));
 
+        Map<File, String> skippedFilesWithReason = new HashMap<>();
+        skippedTestByFilePatternTransformations.forEach((filePattern, reason) -> {
+            //resolve file pattern to concrete files
+            for (File file : getTestFiles().matching(filePattern).getFiles()) {
+                skippedFilesWithReason.put(file, reason);
+            }
+        });
+
+        Map<File, List<Pair<String, String>>> skippedFilesWithTestAndReason = new HashMap<>();
+        skippedTestByTestNameTransformations.forEach((filePattern, testWithReason) -> {
+            //resolve file pattern to concrete files
+            for (File file : getTestFiles().matching(filePattern).getFiles()) {
+                skippedFilesWithTestAndReason.put(file, testWithReason);
+            }
+        });
+
         RestTestTransformer transformer = new RestTestTransformer();
         // TODO: instead of flattening the FileTree here leverage FileTree.visit() so we can preserve folder hierarchy in a more robust way
         for (File file : getTestFiles().getFiles()) {
             YAMLParser yamlParser = YAML_FACTORY.createParser(file);
             List<ObjectNode> tests = READER.<ObjectNode>readValues(yamlParser).readAll();
-            List<ObjectNode> transformRestTests = transformer.transformRestTests(new LinkedList<>(tests), transformations);
+            List<ObjectNode> transformRestTests;
+            if (skippedFilesWithReason.containsKey(file)) {
+                //skip all the tests in the file
+                transformRestTests = transformer.transformRestTests(new LinkedList<>(tests),
+                    Collections.singletonList(new Skip(skippedFilesWithReason.get(file))));
+            } else {
+                if (skippedFilesWithTestAndReason.containsKey(file)) {
+                    //skip the named tests for this file
+                    skippedFilesWithTestAndReason.get(file).forEach(fullTestNameAndReasonPair -> {
+                        String prefix = file.getName().replace(".yml", "/");
+                        String singleTestName = fullTestNameAndReasonPair.getLeft().replaceAll(".*" + prefix, "");
+                        transformations.add(new Skip(singleTestName, fullTestNameAndReasonPair.getRight()));
+                    });
+                }
+                transformRestTests = transformer.transformRestTests(new LinkedList<>(tests), transformations);
+            }
+
             // convert to url to ensure forward slashes
             String[] testFileParts = file.toURI().toURL().getPath().split(REST_TEST_PREFIX);
             if (testFileParts.length != 2) {
@@ -397,4 +485,19 @@ public class RestCompatTestTransformTask extends DefaultTask {
     public List<RestTestTransform<?>> getTransformations() {
         return transformations;
     }
+
+    @Input
+    public String getSkippedTestByFilePatternTransformations() {
+        return skippedTestByFilePatternTransformations.keySet().stream()
+            .map(key -> String.join(",", key.getIncludes()) + skippedTestByFilePatternTransformations.get(key))
+            .collect(Collectors.joining());
+    }
+
+    @Input
+    public String getSkippedTestByTestNameTransformations() {
+
+        return skippedTestByTestNameTransformations.keySet().stream()
+            .map(key -> String.join(",", key.getIncludes()) + skippedTestByTestNameTransformations.get(key))
+            .collect(Collectors.joining());
+    }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/skip/Skip.java

@@ -0,0 +1,117 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.test.rest.transform.skip;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.databind.node.TextNode;
+
+import org.elasticsearch.gradle.internal.test.rest.transform.RestTestTransform;
+import org.elasticsearch.gradle.internal.test.rest.transform.RestTestTransformByParentObject;
+import org.elasticsearch.gradle.internal.test.rest.transform.RestTestTransformGlobalSetup;
+import org.elasticsearch.gradle.internal.test.rest.transform.feature.FeatureInjector;
+import org.gradle.api.tasks.Input;
+import org.gradle.api.tasks.Internal;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
+
+/**
+ * A {@link RestTestTransform} that injects a skip into a REST test.
+ */
+public class Skip implements RestTestTransformGlobalSetup, RestTestTransformByParentObject {
+
+    private static JsonNodeFactory jsonNodeFactory = JsonNodeFactory.withExactBigDecimals(false);
+
+    private final String skipReason;
+    private final String testName;
+
+    public Skip(String testName, String skipReason) {
+        this.skipReason = skipReason;
+        this.testName = testName;
+    }
+
+    public Skip(String skipReason) {
+        this.skipReason = skipReason;
+        this.testName = "";
+    }
+
+    @Override
+    public ObjectNode transformSetup(@Nullable ObjectNode setupNodeParent) {
+        // only transform the global setup if there is no named test
+        if (testName.isBlank()) {
+            ArrayNode setupNode;
+            if (setupNodeParent == null) {
+                setupNodeParent = new ObjectNode(jsonNodeFactory);
+                setupNode = new ArrayNode(jsonNodeFactory);
+                setupNodeParent.set("setup", setupNode);
+            }
+            setupNode = (ArrayNode) setupNodeParent.get("setup");
+            addSkip(setupNode);
+        }
+        return setupNodeParent;
+    }
+
+    private void addSkip(ArrayNode skipParent) {
+        Iterator<JsonNode> skipParentIt = skipParent.elements();
+        boolean found = false;
+        while (skipParentIt.hasNext()) {
+            JsonNode arrayEntry = skipParentIt.next();
+            if (arrayEntry.isObject()) {
+                ObjectNode skipCandidate = (ObjectNode) arrayEntry;
+                if (skipCandidate.get("skip") != null) {
+                    ObjectNode skipNode = (ObjectNode) skipCandidate.get("skip");
+                    skipNode.replace("version", TextNode.valueOf("all"));
+                    skipNode.replace("reason", TextNode.valueOf(skipReason));
+                    found = true;
+                    break;
+                }
+            }
+
+        }
+
+        if (found == false) {
+            ObjectNode skipNode = new ObjectNode(jsonNodeFactory);
+            skipParent.insert(0, skipNode);
+            ObjectNode skipChild = new ObjectNode(jsonNodeFactory);
+            skipChild.set("version", TextNode.valueOf("all"));
+            skipChild.set("reason", TextNode.valueOf(skipReason));
+            skipNode.set("skip", skipChild);
+        }
+    }
+
+
+    @Override
+    public void transformTest(ObjectNode parent) {
+        if (testName.isBlank() == false) {
+            assert parent.get(testName) instanceof ArrayNode;
+            addSkip((ArrayNode) parent.get(testName));
+        }
+    }
+
+    @Override
+    public String getKeyToFind() {
+        return testName;
+    }
+
+    @Input
+    public String getSkipReason() {
+        return skipReason;
+    }
+
+    @Input
+    public String getTestName() {
+        return testName;
+    }
+}

build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/skip/SkipTests.java

@@ -0,0 +1,105 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.test.rest.transform.skip;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+import org.elasticsearch.gradle.internal.test.rest.transform.AssertObjectNodes;
+import org.elasticsearch.gradle.internal.test.rest.transform.RestTestTransform;
+import org.elasticsearch.gradle.internal.test.rest.transform.TransformTests;
+import org.elasticsearch.gradle.internal.test.rest.transform.match.ReplaceKeyInMatch;
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.List;
+
+public class SkipTests extends TransformTests {
+
+
+    @Test
+    public void testAddGlobalSetup() throws Exception {
+        String test_original = "/rest/transform/skip/without_setup_original.yml";
+        List<ObjectNode> tests = getTests(test_original);
+
+        String test_transformed = "/rest/transform/skip/without_setup_transformed.yml";
+        List<ObjectNode> expectedTransformation = getTests(test_transformed);
+
+        List<ObjectNode> transformedTests = transformTests(
+            tests,
+            Collections.singletonList(new Skip("my reason"))
+        );
+
+        AssertObjectNodes.areEqual(transformedTests, expectedTransformation);
+    }
+
+    @Test
+    public void testModifyGlobalSetupWithSkip() throws Exception {
+        String test_original = "/rest/transform/skip/without_setup_original.yml";
+        List<ObjectNode> tests = getTests(test_original);
+
+        String test_transformed = "/rest/transform/skip/without_setup_transformed.yml";
+        List<ObjectNode> expectedTransformation = getTests(test_transformed);
+
+        List<ObjectNode> transformedTests = transformTests(
+            tests,
+            Collections.singletonList(new Skip("my reason"))
+        );
+
+        AssertObjectNodes.areEqual(transformedTests, expectedTransformation);
+    }
+
+    @Test
+    public void testModifyGlobalSetupWithoutSkip() throws Exception {
+        String test_original = "/rest/transform/skip/with_setup_no_skip_original.yml";
+        List<ObjectNode> tests = getTests(test_original);
+
+        String test_transformed = "/rest/transform/skip/with_setup_no_skip_transformed.yml";
+        List<ObjectNode> expectedTransformation = getTests(test_transformed);
+
+        List<ObjectNode> transformedTests = transformTests(
+            tests,
+            Collections.singletonList(new Skip("my reason"))
+        );
+
+        AssertObjectNodes.areEqual(transformedTests, expectedTransformation);
+    }
+
+    @Test
+    public void testModifyGlobalSetupWithFeatures() throws Exception {
+        String test_original = "/rest/transform/skip/with_features_original.yml";
+        List<ObjectNode> tests = getTests(test_original);
+
+        String test_transformed = "/rest/transform/skip/with_features_transformed.yml";
+        List<ObjectNode> expectedTransformation = getTests(test_transformed);
+
+        List<ObjectNode> transformedTests = transformTests(
+            tests,
+            Collections.singletonList(new Skip("my reason"))
+        );
+
+        AssertObjectNodes.areEqual(transformedTests, expectedTransformation);
+    }
+
+    @Test
+    public void testModifyPerTestSetup() throws Exception {
+        String test_original = "/rest/transform/skip/per_test_original.yml";
+        List<ObjectNode> tests = getTests(test_original);
+
+        String test_transformed = "/rest/transform/skip/per_test_transformed.yml";
+        List<ObjectNode> expectedTransformation = getTests(test_transformed);
+
+        List<ObjectNode> transformedTests = transformTests(
+            tests,
+            List.of(new Skip("Two Test", "my reason"), new Skip("Three Test", "another reason"))
+        );
+
+        AssertObjectNodes.areEqual(transformedTests, expectedTransformation);
+    }
+
+}

build-tools-internal/src/test/resources/rest/transform/skip/per_test_original.yml

@@ -0,0 +1,18 @@
+---
+"One Test":
+  - do:
+      something:
+        id: "something1"
+  - match: { acknowledged: true }
+---
+"Two Test":
+  - do:
+      something:
+        id: "something2"
+  - match: { acknowledged: true }
+---
+"Three Test":
+  - do:
+      something:
+        id: "something3"
+  - match: { acknowledged: true }

build-tools-internal/src/test/resources/rest/transform/skip/per_test_transformed.yml

@@ -0,0 +1,24 @@
+---
+"One Test":
+  - do:
+      something:
+        id: "something1"
+  - match: { acknowledged: true }
+---
+"Two Test":
+  - skip:
+      version: "all"
+      reason: "my reason"
+  - do:
+      something:
+        id: "something2"
+  - match: { acknowledged: true }
+---
+"Three Test":
+  - skip:
+      version: "all"
+      reason: "another reason"
+  - do:
+      something:
+        id: "something3"
+  - match: { acknowledged: true }

build-tools-internal/src/test/resources/rest/transform/skip/with_features_original.yml

@@ -0,0 +1,13 @@
+---
+setup:
+  - skip:
+      features:
+      - pre_existing_feature1
+      - pre_existing_feature2
+---
+"Test with multiple feature setup":
+  - do:
+      something:
+        id: "something"
+  - match: { acknowledged: true }
+

build-tools-internal/src/test/resources/rest/transform/skip/with_features_transformed.yml

@@ -0,0 +1,15 @@
+---
+setup:
+  - skip:
+      features:
+        - pre_existing_feature1
+        - pre_existing_feature2
+      version: "all"
+      reason: "my reason"
+---
+"Test with multiple feature setup":
+  - do:
+      something:
+        id: "something"
+  - match: { acknowledged: true }
+

build-tools-internal/src/test/resources/rest/transform/skip/with_setup_no_skip_original.yml

@@ -0,0 +1,13 @@
+---
+setup:
+  - do:
+      some.setup:
+        index: blah
+---
+"Test with setup but no skip (and by inference no features)":
+  - do:
+      something:
+        id: "something"
+  - match: { acknowledged: true }
+
+

build-tools-internal/src/test/resources/rest/transform/skip/with_setup_no_skip_transformed.yml

@@ -0,0 +1,16 @@
+---
+setup:
+  - skip:
+      version: "all"
+      reason: "my reason"
+  - do:
+      some.setup:
+        index: blah
+---
+"Test with setup but no skip (and by inference no features)":
+  - do:
+      something:
+        id: "something"
+  - match: { acknowledged: true }
+
+

build-tools-internal/src/test/resources/rest/transform/skip/with_skip_original.yml

@@ -0,0 +1,13 @@
+---
+setup:
+  - skip:
+      version: " - 7.1.99"
+      reason:  why not
+---
+"Test with setup and skip but no feature":
+  - do:
+      something:
+        id: "something"
+  - match: { acknowledged: true }
+
+

build-tools-internal/src/test/resources/rest/transform/skip/with_skip_transformed.yml

@@ -0,0 +1,13 @@
+---
+setup:
+  - skip:
+      version: "all"
+      reason:  "my reason"
+---
+"Test with setup and skip but no feature":
+  - do:
+      something:
+        id: "something"
+  - match: { acknowledged: true }
+
+

build-tools-internal/src/test/resources/rest/transform/skip/without_setup_original.yml

@@ -0,0 +1,7 @@
+---
+"Test without a setup":
+  - do:
+      something:
+        id: "something"
+  - match: { acknowledged: true }
+

build-tools-internal/src/test/resources/rest/transform/skip/without_setup_transformed.yml

@@ -0,0 +1,11 @@
+setup:
+  - skip:
+      version: "all"
+      reason: "my reason"
+---
+"Test without a setup":
+  - do:
+      something:
+        id: "something"
+  - match: { acknowledged: true }
+

modules/analysis-common/build.gradle

@@ -25,12 +25,11 @@ dependencies {
   compileOnly project(':modules:lang-painless')
 }
 
-tasks.named("yamlRestTestV7CompatTest").configure {
-  systemProperty 'tests.rest.blacklist', [
-       //marked as not needing compatible api
-       'indices.analyze/10_analyze/htmlStrip_deprecated', // Cleanup versioned deprecations in analysis #41560
-       'analysis-common/40_token_filters/delimited_payload_filter_error', //Remove preconfigured delimited_payload_filter #43686
-       'analysis-common/20_analyzers/standard_html_strip', // Cleanup versioned deprecations in analysis #41560
-       'search.query/50_queries_with_synonyms/Test common terms query with stacked tokens', // #42654 -  `common` query throws an exception
-    ].join(',')
+
+tasks.named("yamlRestTestV7CompatTransform").configure { task ->
+  task.skipTest("indices.analyze/10_analyze/htmlStrip_deprecated", "Cleanup versioned deprecations in analysis #41560")
+  task.skipTest("analysis-common/40_token_filters/delimited_payload_filter_error", "Remove preconfigured delimited_payload_filter #43686")
+  task.skipTest("analysis-common/20_analyzers/standard_html_strip", "Cleanup versioned deprecations in analysis #41560")
+  task.skipTest("search.query/50_queries_with_synonyms/Test common terms query with stacked tokens", "#42654 -  `common` query throws an exception")
 }
+

modules/percolator/build.gradle

@@ -24,6 +24,6 @@ restResources {
   }
 }
 
-tasks.named("yamlRestTestV7CompatTransform").configure({ task ->
+tasks.named("yamlRestTestV7CompatTransform").configure{ task ->
   task.addAllowedWarningRegex("\\[types removal\\].*")
-})
+}

modules/reindex/build.gradle

@@ -155,13 +155,10 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) {
     }
   }
 }
-tasks.named("yamlRestTestV7CompatTransform").configure({ task ->
-  task.addAllowedWarningRegex("\\[types removal\\].*")
-})
-
-tasks.named("yamlRestTestV7CompatTest").configure {
-      systemProperty 'tests.rest.blacklist', [
-          'reindex/20_validation/reindex without source gives useful error message', // exception with a type. Not much benefit adding _doc there.
-          'update_by_query/20_validation/update_by_query without source gives useful error message' // exception with a type. Not much benefit adding _doc there.
-        ].join(',')
+
+tasks.named("yamlRestTestV7CompatTransform").configure { task ->
+    task.skipTest("reindex/20_validation/reindex without source gives useful error message", "exception with a type. Not much benefit adding _doc there.")
+    task.skipTest("update_by_query/20_validation/update_by_query without source gives useful error message", "exception with a type. Not much benefit adding _doc there.")
+    task.addAllowedWarningRegex("\\[types removal\\].*")
 }
+

plugins/analysis-icu/build.gradle

@@ -34,9 +34,8 @@ restResources {
 tasks.named("dependencyLicenses").configure {
   mapping from: /lucene-.*/, to: 'lucene'
 }
-tasks.named("yamlRestTestV7CompatTest").configure {
-  systemProperty 'tests.rest.blacklist', [
-      //marked as not needing compatible api
-      'analysis_icu/10_basic/Normalization with deprecated unicodeSetFilter' // Cleanup versioned deprecations in analysis #41560
-    ].join(',')
+
+tasks.named("yamlRestTestV7CompatTransform").configure { task ->
+    task.skipTest("analysis_icu/10_basic/Normalization with deprecated unicodeSetFilter", "Cleanup versioned deprecations in analysis #41560")
 }
+

rest-api-spec/build.gradle

@@ -42,71 +42,46 @@ testClusters.configureEach {
 
 tasks.named("test").configure { enabled = false }
 tasks.named("jarHell").configure { enabled = false }
-tasks.named("yamlRestTestV7CompatTest").configure {
-  systemProperty 'tests.rest.blacklist',  [
-            // Cat API are meant to be consumed by humans, so will not be supported by Compatible REST API
-            'cat*/*/*',
-            // type information about the type is removed and not passed down. The logic to check for this is also removed.
-            'delete/70_mix_typeless_typeful/DELETE with typeless API on an index that has types',
-            // WILL NOT BE FIXED - failing due to not recognising missing type (the type path param is ignored)
-            'get/100_mix_typeless_typeful/GET with typeless API on an index that has types',
-            // type information about the type is removed and not passed down. The logic to check for this is also removed.
-            'indices.create/20_mix_typeless_typeful/Implicitly create a typed index while there is a typeless template',
-            'indices.create/20_mix_typeless_typeful/Implicitly create a typeless index while there is a typed template',
-            //
-            // This test returns test_index.mappings:{} when {} was expected. difference between 20_missing_field and 21_missing_field_with_types?
-            'indices.get_field_mapping/21_missing_field_with_types/Return empty object if field doesn\'t exist, but type and index do',
-            // The information about the type is not present in the index. hence it cannot know if the type exist or not.
-            'indices.get_field_mapping/30_missing_type/Raise 404 when type doesn\'t exist',
-            // The information about the type is not present in the index. hence it cannot know if the type exist or not.
-            'indices.get_mapping/20_missing_type/Existent and non-existent type returns 404 and the existing type',
-            'indices.get_mapping/20_missing_type/Existent and non-existent types returns 404 and the existing type',
-            'indices.get_mapping/20_missing_type/No type matching pattern returns 404',
-            'indices.get_mapping/20_missing_type/Non-existent type returns 404',
-            'indices.get_mapping/20_missing_type/Type missing when no types exist',
-            //
-            // The information about the type is not present in the index. hence it cannot know if the type was already used or not
-            'indices.put_mapping/20_mix_typeless_typeful/PUT mapping with _doc on an index that has types',
-            'indices.put_mapping/20_mix_typeless_typeful/PUT mapping with typeless API on an index that has types',
-            // there is a small distinction between empty mappings and no mappings at all. The code to implement this test was refactored #54003
-            // field search on _type field- not implementing. The data for _type is considered incorrect in this search
-            'search/160_exists_query/Test exists query on _type field',
-            //type information is not stored, hence the the index will be found
-            'termvectors/50_mix_typeless_typeful/Term vectors with typeless API on an index that has types',
-            // mget - these use cases are no longer valid, because we always default to _doc.
-            // This mean test cases where there is assertion on not finging by type won't work
-            'mget/11_default_index_type/Default index/type',
-            'mget/16_basic_with_types/Basic multi-get',
-            // asserting about type not found won't work as we ignore the type information
-            'explain/40_mix_typeless_typeful/Explain with typeless API on an index that has types',
-            // translog settings removal is not supported under compatible api
-            'indices.stats/20_translog/Translog retention settings are deprecated',
-            'indices.stats/20_translog/Translog retention without soft_deletes',
-            'indices.stats/20_translog/Translog stats on closed indices without soft-deletes',
 
-            // field usage results will be different between lucene versions
-            'indices.stats/60_field_usage/*',
+tasks.named("yamlRestTestV7CompatTransform").configure { task ->
+
+  task.skipTestsByFilePattern("**/cat*/*.yml", "Cat API are meant to be consumed by humans, so will not be supported by Compatible REST API")
+  task.skipTestsByFilePattern("**/indices.upgrade/*.yml", "upgrade api will only get a dummy endpoint returning an exception suggesting to use _reindex")
+  task.skipTestsByFilePattern("**/indices.stats/60_field_usage/*/*.yml", "field usage results will be different between lucene versions")
+
+  task.skipTest("indices.create/20_mix_typeless_typeful/Implicitly create a typed index while there is a typeless template", "Type information about the type is removed and not passed down. The logic to check for this is also removed.")
+  task.skipTest("indices.create/20_mix_typeless_typeful/Implicitly create a typeless index while there is a typed template", "Type information about the type is removed and not passed down. The logic to check for this is also removed.")
+  task.skipTest("delete/70_mix_typeless_typeful/DELETE with typeless API on an index that has types", "Type information about the type is removed and not passed down. The logic to check for this is also removed.");
+  task.skipTest("get/100_mix_typeless_typeful/GET with typeless API on an index that has types", "Failing due to not recognising missing type (the type path param is ignored, will no be fixed");
+  task.skipTest("indices.get_field_mapping/21_missing_field_with_types/Return empty object if field doesn't exist, but type and index do", "This test returns test_index.mappings:{} when {} was expected. difference between 20_missing_field and 21_missing_field_with_types?")
+  task.skipTest("indices.get_field_mapping/30_missing_type/Raise 404 when type doesn't exist", "The information about the type is not present in the index. hence it cannot know if the type exist or not.")
+  task.skipTest("indices.get_mapping/20_missing_type/Existent and non-existent type returns 404 and the existing type", " The information about the type is not present in the index. hence it cannot know if the type exist or not")
+  task.skipTest("indices.get_mapping/20_missing_type/Existent and non-existent types returns 404 and the existing type", "The information about the type is not present in the index. hence it cannot know if the type exist or not.")
+  task.skipTest("indices.get_mapping/20_missing_type/No type matching pattern returns 404", "The information about the type is not present in the index. hence it cannot know if the type exist or not.")
+  task.skipTest("indices.get_mapping/20_missing_type/Non-existent type returns 404", "The information about the type is not present in the index. hence it cannot know if the type exist or not.")
+  task.skipTest("indices.get_mapping/20_missing_type/Type missing when no types exist", "The information about the type is not present in the index. hence it cannot know if the type exist or not.")
+  task.skipTest("indices.put_mapping/20_mix_typeless_typeful/PUT mapping with _doc on an index that has types", "The information about the type is not present in the index. hence it cannot know if the type was already used or not")
+  task.skipTest("indices.put_mapping/20_mix_typeless_typeful/PUT mapping with typeless API on an index that has types", "The information about the type is not present in the index. hence it cannot know if the type was already used or not")
+  task.skipTest("search/160_exists_query/Test exists query on _type field", "There is a small distinction between empty mappings and no mappings at all. The code to implement this test was refactored #54003; field search on _type field- not implementing. The data for _type is considered incorrect in this search")
+  task.skipTest("termvectors/50_mix_typeless_typeful/Term vectors with typeless API on an index that has types", "type information is not stored, hence the the index will be found")
+  task.skipTest("mget/11_default_index_type/Default index/type", "mget - these use cases are no longer valid because we always default to _doc.;  This mean test cases where there is assertion on not finding by type won't work")
+  task.skipTest("mget/16_basic_with_types/Basic multi-get", "mget - these use cases are no longer valid, because we always default to _doc.;  This mean test cases where there is assertion on not finding by type won't work")
+  task.skipTest("explain/40_mix_typeless_typeful/Explain with typeless API on an index that has types", "asserting about type not found won't work as we ignore the type information")
+  task.skipTest("indices.stats/20_translog/Translog retention settings are deprecated", "translog settings removal is not supported under compatible api")
+  task.skipTest("indices.stats/20_translog/Translog retention without soft_deletes", "translog settings removal is not supported under compatible api")
+  task.skipTest("indices.stats/20_translog/Translog stats on closed indices without soft-deletes", "translog settings removal is not supported under compatible api")
+  task.skipTest("search.aggregation/370_doc_count_field/Test filters agg with doc_count", "Uses profiler for assertions which is not backwards compatible")
+  task.skipTest("indices.create/10_basic/Create index without soft deletes", "Make soft-deletes mandatory in 8.0 #51122 - settings changes are note supported in Rest Api compatibility")
+  task.skipTest("field_caps/30_filter/Field caps with index filter", "behaviour change after #63692 4digits dates are parsed as epoch and in quotes as year")
+  task.skipTest("indices.forcemerge/10_basic/Check deprecation warning when incompatible only_expunge_deletes and max_num_segments values are both set", "#44761 bug fix")
+  task.skipTest("search/340_type_query/type query", "#47207 type query throws exception in compatible mode")
+  task.skipTest("search.aggregation/200_top_hits_metric/top_hits aggregation with sequence numbers", "#42809 the use nested path and filter sort throws an exception")
+  task.skipTest("search/310_match_bool_prefix/multi_match multiple fields with cutoff_frequency throws exception", "#42654 cutoff_frequency, common terms are not supported. Throwing an exception")
+  task.skipTest("search.aggregation/20_terms/string profiler via global ordinals filters implementation", "The profiler results aren't backwards compatible.")
+  task.skipTest("search.aggregation/20_terms/string profiler via global ordinals native implementation", "The profiler results aren't backwards compatible.")
+  task.skipTest("search.aggregation/20_terms/string profiler via map", "The profiler results aren't backwards compatible.")
+  task.skipTest("search.aggregation/20_terms/numeric profiler", "The profiler results aren't backwards compatible.")
 
-            // upgrade api will only get a dummy endpoint returning an exception suggesting to use _reindex
-            'indices.upgrade/*/*',
-
-
-            'search.aggregation/20_terms/*profiler*', // The profiler results aren't backwards compatible.
-            'search.aggregation/370_doc_count_field/Test filters agg with doc_count', // Uses profiler for assertions which is not backwards compatible
-
-            'indices.create/10_basic/Create index without soft deletes', //Make soft-deletes mandatory in 8.0 #51122 - settings changes are note supported in Rest Api compatibility
-
-            'field_caps/30_filter/Field caps with index filter', //behaviour change after #63692 4digits dates are parsed as epoch and in quotes as year
-
-            'indices.forcemerge/10_basic/Check deprecation warning when incompatible only_expunge_deletes and max_num_segments values are both set', //#44761 bug fix,
-
-            'search/340_type_query/type query', //#47207 type query throws exception in compatible mode
-            'search.aggregation/200_top_hits_metric/top_hits aggregation with sequence numbers', // #42809 the use nested path and filter sort throws an exception
-            'search/310_match_bool_prefix/multi_match multiple fields with cutoff_frequency throws exception', //#42654 cutoff_frequency, common terms are not supported. Throwing an exception
-    ].join(',')
-}
-
-tasks.named("yamlRestTestV7CompatTransform").configure({ task ->
   task.replaceValueInMatch("_type", "_doc")
   task.addAllowedWarningRegex("\\[types removal\\].*")
   task.replaceValueInMatch("nodes.\$node_id.roles.8", "ml", "node_info role test")
@@ -241,7 +216,7 @@ tasks.named("yamlRestTestV7CompatTransform").configure({ task ->
   // sync_id is no longer available in SegmentInfos.userData // "indices.flush/10_basic/Index synced flush rest test"
   task.replaceIsTrue("indices.testing.shards.0.0.commit.user_data.sync_id", "indices.testing.shards.0.0.commit.user_data")
 
-})
+}
 
 tasks.register('enforceYamlTestConvention').configure {
   doLast {

x-pack/plugin/build.gradle

@@ -90,7 +90,32 @@ tasks.named("yamlRestTest").configure {
   dependsOn "copyExtraResources"
 }
 
-tasks.named("yamlRestTestV7CompatTransform").configure({ task ->
+tasks.named("yamlRestTestV7CompatTransform").configure{ task ->
+  task.skipTest("vectors/10_dense_vector_basic/Deprecated function signature", "to support it, it would require to almost revert back the #48725 and complicate the code" )
+  task.skipTest("vectors/30_sparse_vector_basic/Cosine Similarity", "not supported for compatibility")
+  task.skipTest("vectors/30_sparse_vector_basic/Deprecated function signature", "not supported for compatibility")
+  task.skipTest("vectors/30_sparse_vector_basic/Dot Product", "not supported for compatibility")
+  task.skipTest("vectors/35_sparse_vector_l1l2/L1 norm", "not supported for compatibility")
+  task.skipTest("vectors/35_sparse_vector_l1l2/L2 norm", "not supported for compatibility")
+  task.skipTest("vectors/40_sparse_vector_special_cases/Dimensions can be sorted differently", "not supported for compatibility")
+  task.skipTest("vectors/40_sparse_vector_special_cases/Documents missing a vector field", "not supported for compatibility")
+  task.skipTest("vectors/40_sparse_vector_special_cases/Query vector has different dimensions from documents' vectors", "not supported for compatibility")
+  task.skipTest("vectors/40_sparse_vector_special_cases/Sparse vectors should error with dense vector functions", "not supported for compatibility")
+  task.skipTest("vectors/40_sparse_vector_special_cases/Vectors of different dimensions and data types", "not supported for compatibility")
+  task.skipTest("vectors/50_vector_stats/Usage stats on vector fields", "not supported for compatibility")
+  task.skipTest("roles/30_prohibited_role_query/Test use prohibited query inside role query", "put role request with a term lookup (deprecated) and type. Requires validation in REST layer")
+  task.skipTest("ml/jobs_crud/Test create job with delimited format", "removing undocumented functionality")
+  task.skipTest("ml/datafeeds_crud/Test update datafeed to point to missing job", "behaviour change #44752 - not allowing to update datafeed job_id")
+  task.skipTest("ml/datafeeds_crud/Test update datafeed to point to different job", "behaviour change #44752 - not allowing to update datafeed job_id")
+  task.skipTest("ml/datafeeds_crud/Test update datafeed to point to job already attached to another datafeed", "behaviour change #44752 - not allowing to update datafeed job_id")
+  task.skipTest("rollup/delete_job/Test basic delete_job", "rollup was an experimental feature, also see #41227")
+  task.skipTest("rollup/delete_job/Test delete job twice", "rollup was an experimental feature, also see #41227")
+  task.skipTest("rollup/delete_job/Test delete running job", "rollup was an experimental feature, also see #41227")
+  task.skipTest("rollup/get_jobs/Test basic get_jobs", "rollup was an experimental feature, also see #41227")
+  task.skipTest("rollup/put_job/Test basic put_job", "rollup was an experimental feature, also see #41227")
+  task.skipTest("rollup/start_job/Test start job twice", "rollup was an experimental feature, also see #41227")
+  task.skipTest("ml/trained_model_cat_apis/Test cat trained models", "A type field was added to cat.ml_trained_models #73660, this is a backwards compatible change. Still this is a cat api, and we don't support them with rest api compatibility. (the test would be very hard to transform too)")
+
   task.replaceKeyInDo("license.delete", "xpack-license.delete")
   task.replaceKeyInDo("license.get", "xpack-license.get")
   task.replaceKeyInDo("license.get_basic_status", "xpack-license.get_basic_status")
@@ -164,50 +189,9 @@ tasks.named("yamlRestTestV7CompatTransform").configure({ task ->
   task.replaceValueInMatch("_type", "_doc")
   task.addAllowedWarningRegex("\\[types removal\\].*")
   task.addAllowedWarningRegexForTest("Including \\[accept_enterprise\\] in get license.*", "Installing enterprise license")
-})
-tasks.named("yamlRestTestV7CompatTest").configure {
-  systemProperty 'tests.rest.blacklist', [
-      // to support it, it would require to almost revert back the #48725 and complicate the code
-      'vectors/10_dense_vector_basic/Deprecated function signature',
-      // not going to be supported
-      'vectors/30_sparse_vector_basic/Cosine Similarity',
-      'vectors/30_sparse_vector_basic/Deprecated function signature',
-      'vectors/30_sparse_vector_basic/Dot Product',
-      'vectors/35_sparse_vector_l1l2/L1 norm',
-      'vectors/35_sparse_vector_l1l2/L2 norm',
-      'vectors/40_sparse_vector_special_cases/Dimensions can be sorted differently',
-      'vectors/40_sparse_vector_special_cases/Documents missing a vector field',
-      'vectors/40_sparse_vector_special_cases/Query vector has different dimensions from documents\' vectors',
-      'vectors/40_sparse_vector_special_cases/Sparse vectors should error with dense vector functions',
-      'vectors/40_sparse_vector_special_cases/Vectors of different dimensions and data types',
-      // the test uses sparse vector - not supported
-      'vectors/50_vector_stats/Usage stats on vector fields',
-
-      // put role request with a term lookup (deprecated) and type. Requires validation in REST layer
-      'roles/30_prohibited_role_query/Test use prohibited query inside role query',
-      //removing undocumented functionality
-      'ml/jobs_crud/Test create job with delimited format',
-      // behaviour change #44752 - not allowing to update datafeed job_id
-      'ml/datafeeds_crud/Test update datafeed to point to missing job',
-      'ml/datafeeds_crud/Test update datafeed to point to different job',
-      'ml/datafeeds_crud/Test update datafeed to point to job already attached to another datafeed',
-      //rollup was an experimental feature
-      //https://github.com/elastic/elasticsearch/pull/41227.
-      'rollup/delete_job/Test basic delete_job',
-      'rollup/delete_job/Test delete job twice',
-      'rollup/delete_job/Test delete running job',
-      'rollup/get_jobs/Test basic get_jobs',
-      'rollup/put_job/Test basic put_job',
-      //https://github.com/elastic/elasticsearch/pull/41502
-      'rollup/start_job/Test start job twice',
-
-      // a type field was added to cat.ml_trained_models #73660, this is a backwards compatible change.
-      // still this is a cat api, and we don't support them with rest api compatibility. (the test would be very hard to transform too)
-      'ml/trained_model_cat_apis/Test cat trained models'
-  ].join(',')
-  dependsOn "copyExtraResources"
 }
 
+
 testClusters.configureEach {
   testDistribution = 'DEFAULT' // this is important since we use the reindex module in ML
   setting 'xpack.ml.enabled', 'true'

x-pack/plugin/watcher/qa/rest/build.gradle

@@ -35,16 +35,11 @@ if (BuildParams.inFipsJvm){
   tasks.named("yamlRestTest").configure{enabled = false }
 }
 
-tasks.named("yamlRestTestV7CompatTest").configure {
-    systemProperty 'tests.rest.blacklist', [
-            // remove JodaCompatibleDateTime -- ZonedDateTime doesn't output millis/nanos if they're 0 (#78417)
-            'mustache/30_search_input/Test search input mustache integration (using request body and rest_total_hits_as_int)',
-            'mustache/30_search_input/Test search input mustache integration (using request body)',
-            'mustache/40_search_transform/Test search transform mustache integration (using request body)'
-    ].join(',')
-}
 
-tasks.named("yamlRestTestV7CompatTransform").configure({ task ->
+tasks.named("yamlRestTestV7CompatTransform").configure{ task ->
+  task.skipTest("mustache/30_search_input/Test search input mustache integration (using request body and rest_total_hits_as_int)", "remove JodaCompatibleDateTime -- ZonedDateTime doesn't output millis/nanos if they're 0 (#78417)")
+  task.skipTest("mustache/30_search_input/Test search input mustache integration (using request body)", "remove JodaCompatibleDateTime -- ZonedDateTime doesn't output millis/nanos if they're 0 (#78417)")
+  task.skipTest("mustache/40_search_transform/Test search transform mustache integration (using request body)", "remove JodaCompatibleDateTime -- ZonedDateTime doesn't output millis/nanos if they're 0 (#78417)")
   task.replaceKeyInDo("watcher.ack_watch", "xpack-watcher.ack_watch")
   task.replaceKeyInDo("watcher.activate_watch", "xpack-watcher.activate_watch")
   task.replaceKeyInDo("watcher.deactivate_watch", "xpack-watcher.deactivate_watch")
@@ -60,5 +55,4 @@ tasks.named("yamlRestTestV7CompatTransform").configure({ task ->
   task.addAllowedWarningRegex("\\[types removal\\].*")
   task.replaceValueTextByKeyValue("path", "/my_index/my_type/{{ctx.watch_id}}", "/my_index/_doc/{{ctx.watch_id}}",
     "Test webhook action with mustache integration")
-
-})
+}