Introduce yamlRestCompatTest task and plugin. (#62985)

This commit introduces a plugin to run REST compatibility tests.
The plugin is not currently applied to any projects with this commit, 
but it establishes the foundation for testing REST compatibility. 

This plugin will be applied to all core, plugin, and module projects 
that currently have YAML based REST tests. This plugin, when applied
will do the following:

* Create a new sourceset that extends from the yamlRestTest sourceset
* Ensure the default distribution is used to execute the tests
* Checkout bwc:minor version of the source code
* Copy the rest api and rest tests from bwc:minor to the new sourceset
* Create the test task, but manipulate the classpath so it uses the 
  "normal" YAML test runner, but the "compat" set of tests
* wire up dependencies, ide, and check

The implementation here depends on :distribution:bwc:minor:checkoutBwcBranch
which is a convenient way to git checkout the most recent prior branch. 
This approach works but is a bit fragile and long term would like a more 
robust way to checkout arbitrary prior branches. This would be a fairly large
effort and is not included in this commit. 

Also not included in this commit is the code necessary to inject the
compatible header, any customization to running the compatibility tests, 
or any of the logic to override specific tests. Future commits will address
those concerns.

buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java

@@ -58,8 +58,10 @@ public class CopyRestApiTask extends DefaultTask {
     final ListProperty<String> includeCore = getProject().getObjects().listProperty(String.class);
     final ListProperty<String> includeXpack = getProject().getObjects().listProperty(String.class);
     String sourceSetName;
+    boolean skipHasRestTestCheck;
     Configuration coreConfig;
     Configuration xpackConfig;
+    Configuration additionalConfig;
 
     private final PatternFilterable corePatternSet;
     private final PatternFilterable xpackPatternSet;
@@ -89,6 +91,11 @@ public class CopyRestApiTask extends DefaultTask {
         return sourceSetName;
     }
 
+    @Input
+    public boolean isSkipHasRestTestCheck() {
+        return skipHasRestTestCheck;
+    }
+
     @SkipWhenEmpty
     @InputFiles
     public FileTree getInputDir() {
@@ -98,7 +105,7 @@ public class CopyRestApiTask extends DefaultTask {
             xpackPatternSet.setIncludes(includeXpack.get().stream().map(prefix -> prefix + "*/**").collect(Collectors.toList()));
             xpackFileTree = xpackConfig.getAsFileTree().matching(xpackPatternSet);
         }
-        boolean projectHasYamlRestTests = projectHasYamlRestTests();
+        boolean projectHasYamlRestTests = skipHasRestTestCheck || projectHasYamlRestTests();
         if (includeCore.get().isEmpty() == false || projectHasYamlRestTests) {
             if (BuildParams.isInternal()) {
                 corePatternSet.setIncludes(includeCore.get().stream().map(prefix -> prefix + "*/**").collect(Collectors.toList()));
@@ -107,7 +114,10 @@ public class CopyRestApiTask extends DefaultTask {
                 coreFileTree = coreConfig.getAsFileTree(); // jar file
             }
         }
-        ConfigurableFileCollection fileCollection = getProject().files(coreFileTree, xpackFileTree);
+
+        ConfigurableFileCollection fileCollection = additionalConfig == null
+            ? getProject().files(coreFileTree, xpackFileTree)
+            : getProject().files(coreFileTree, xpackFileTree, additionalConfig.getAsFileTree());
 
         // if project has rest tests or the includes are explicitly configured execute the task, else NO-SOURCE due to the null input
         return projectHasYamlRestTests || includeCore.get().isEmpty() == false || includeXpack.get().isEmpty() == false
@@ -132,7 +142,7 @@ public class CopyRestApiTask extends DefaultTask {
         if (BuildParams.isInternal()) {
             getLogger().debug("Rest specs for project [{}] will be copied to the test resources.", project.getPath());
             project.copy(c -> {
-                c.from(coreConfig.getSingleFile());
+                c.from(coreConfig.getAsFileTree());
                 c.into(getOutputDir());
                 c.include(corePatternSet.getIncludes());
             });
@@ -164,6 +174,14 @@ public class CopyRestApiTask extends DefaultTask {
                 c.include(xpackPatternSet.getIncludes());
             });
         }
+        // TODO: once https://github.com/elastic/elasticsearch/pull/62968 lands ensure that this uses `getFileSystemOperations()`
+        // copy any additional config
+        if (additionalConfig != null) {
+            project.copy(c -> {
+                c.from(additionalConfig.getAsFileTree());
+                c.into(getOutputDir());
+            });
+        }
     }
 
     /**

buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java

@@ -58,6 +58,7 @@ public class CopyRestTestsTask extends DefaultTask {
     String sourceSetName;
     Configuration coreConfig;
     Configuration xpackConfig;
+    Configuration additionalConfig;
 
     private final PatternFilterable corePatternSet;
     private final PatternFilterable xpackPatternSet;
@@ -104,10 +105,14 @@ public class CopyRestTestsTask extends DefaultTask {
                 coreFileTree = coreConfig.getAsFileTree(); // jar file
             }
         }
-        ConfigurableFileCollection fileCollection = getProject().files(coreFileTree, xpackFileTree);
+        ConfigurableFileCollection fileCollection = additionalConfig == null
+            ? getProject().files(coreFileTree, xpackFileTree)
+            : getProject().files(coreFileTree, xpackFileTree, additionalConfig.getAsFileTree());
 
         // copy tests only if explicitly requested
-        return includeCore.get().isEmpty() == false || includeXpack.get().isEmpty() == false ? fileCollection.getAsFileTree() : null;
+        return includeCore.get().isEmpty() == false || includeXpack.get().isEmpty() == false || additionalConfig != null
+            ? fileCollection.getAsFileTree()
+            : null;
     }
 
     @OutputDirectory
@@ -158,6 +163,14 @@ public class CopyRestTestsTask extends DefaultTask {
                 c.include(xpackPatternSet.getIncludes());
             });
         }
+        // TODO: once https://github.com/elastic/elasticsearch/pull/62968 lands ensure that this uses `getFileSystemOperations()`
+        // copy any additional config
+        if (additionalConfig != null) {
+            project.copy(c -> {
+                c.from(additionalConfig.getAsFileTree());
+                c.into(getOutputDir());
+            });
+        }
     }
 
     private Optional<SourceSet> getSourceSet() {

buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/YamlRestCompatTestPlugin.java

@@ -0,0 +1,189 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle.test.rest;
+
+import org.elasticsearch.gradle.ElasticsearchJavaPlugin;
+import org.elasticsearch.gradle.VersionProperties;
+import org.elasticsearch.gradle.test.RestIntegTestTask;
+import org.elasticsearch.gradle.test.RestTestBasePlugin;
+import org.elasticsearch.gradle.testclusters.ElasticsearchCluster;
+import org.elasticsearch.gradle.testclusters.TestClustersPlugin;
+import org.elasticsearch.gradle.testclusters.TestDistribution;
+import org.elasticsearch.gradle.util.GradleUtils;
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.plugins.JavaBasePlugin;
+import org.gradle.api.provider.Provider;
+import org.gradle.api.tasks.SourceSet;
+import org.gradle.api.tasks.SourceSetContainer;
+
+import java.io.File;
+import java.nio.file.Path;
+
+import static org.elasticsearch.gradle.test.rest.RestTestUtil.createTestCluster;
+import static org.elasticsearch.gradle.test.rest.RestTestUtil.setupDependencies;
+
+/**
+ * Apply this plugin to run the YAML based REST tests from a prior major version against this version's cluster.
+ */
+// TODO: support running tests against multiple prior versions in addition to bwc:minor. To achieve this we will need to create published
+// artifacts that include all of the REST tests for the latest 7.x.y releases
+public class YamlRestCompatTestPlugin implements Plugin<Project> {
+
+    public static final String SOURCE_SET_NAME = "yamlRestCompatTest";
+    private static final Path RELATIVE_API_PATH = Path.of("rest-api-spec/api");
+    private static final Path RELATIVE_TEST_PATH = Path.of("rest-api-spec/test");
+
+    @Override
+    public void apply(Project project) {
+
+        project.getPluginManager().apply(ElasticsearchJavaPlugin.class);
+        project.getPluginManager().apply(TestClustersPlugin.class);
+        project.getPluginManager().apply(RestTestBasePlugin.class);
+        project.getPluginManager().apply(RestResourcesPlugin.class);
+        project.getPluginManager().apply(YamlRestTestPlugin.class);
+
+        RestResourcesExtension extension = project.getExtensions().getByType(RestResourcesExtension.class);
+
+        // create source set
+        SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
+        SourceSet yamlCompatTestSourceSet = sourceSets.create(SOURCE_SET_NAME);
+        SourceSet yamlTestSourceSet = sourceSets.getByName(YamlRestTestPlugin.SOURCE_SET_NAME);
+        GradleUtils.extendSourceSet(project, YamlRestTestPlugin.SOURCE_SET_NAME, SOURCE_SET_NAME);
+
+        // create the test cluster container, and always use the default distribution
+        ElasticsearchCluster testCluster = createTestCluster(project, yamlCompatTestSourceSet);
+        testCluster.setTestDistribution(TestDistribution.DEFAULT);
+
+        // TODO: once https://github.com/elastic/elasticsearch/pull/62473 lands refactor this to reference the checkoutDir as an artifact
+        int priorMajorVersion = VersionProperties.getElasticsearchVersion().getMajor() - 1;
+        final Path checkoutDir = project.findProject(":distribution:bwc:minor")
+            .getBuildDir()
+            .toPath()
+            .resolve("bwc")
+            .resolve("checkout-" + priorMajorVersion + ".x");
+
+        // copy compatible rest specs
+        Configuration compatSpec = project.getConfigurations().create("compatSpec");
+        Configuration xpackCompatSpec = project.getConfigurations().create("xpackCompatSpec");
+        Configuration additionalCompatSpec = project.getConfigurations().create("additionalCompatSpec");
+        Provider<CopyRestApiTask> copyCompatYamlSpecTask = project.getTasks()
+            .register("copyRestApiCompatSpecsTask", CopyRestApiTask.class, task -> {
+                task.includeCore.set(extension.restApi.getIncludeCore());
+                task.includeXpack.set(extension.restApi.getIncludeXpack());
+                task.sourceSetName = SOURCE_SET_NAME;
+                task.skipHasRestTestCheck = true;
+                task.coreConfig = compatSpec;
+                project.getDependencies()
+                    .add(
+                        task.coreConfig.getName(),
+                        project.files(checkoutDir.resolve("rest-api-spec/src/main/resources").resolve(RELATIVE_API_PATH))
+                    );
+                task.xpackConfig = xpackCompatSpec;
+                project.getDependencies()
+                    .add(
+                        task.xpackConfig.getName(),
+                        project.files(checkoutDir.resolve("x-pack/plugin/src/test/resources").resolve(RELATIVE_API_PATH))
+                    );
+                task.additionalConfig = additionalCompatSpec;
+                // per project can define custom specifications
+                project.getDependencies()
+                    .add(
+                        task.additionalConfig.getName(),
+                        project.files(
+                            getCompatProjectPath(project, checkoutDir).resolve("src/yamlRestTest/resources").resolve(RELATIVE_API_PATH)
+                        )
+                    );
+                task.dependsOn(task.coreConfig);
+                task.dependsOn(task.xpackConfig);
+                task.dependsOn(task.additionalConfig);
+                task.dependsOn(":distribution:bwc:minor:checkoutBwcBranch");
+            });
+
+        // copy compatible rest tests
+        Configuration compatTest = project.getConfigurations().create("compatTest");
+        Configuration xpackCompatTest = project.getConfigurations().create("xpackCompatTest");
+        Configuration additionalCompatTest = project.getConfigurations().create("additionalCompatTest");
+        Provider<CopyRestTestsTask> copyCompatYamlTestTask = project.getTasks()
+            .register("copyRestApiCompatTestTask", CopyRestTestsTask.class, task -> {
+                task.includeCore.set(extension.restTests.getIncludeCore());
+                task.includeXpack.set(extension.restTests.getIncludeXpack());
+                task.sourceSetName = SOURCE_SET_NAME;
+                task.coreConfig = compatTest;
+                project.getDependencies()
+                    .add(
+                        task.coreConfig.getName(),
+                        project.files(checkoutDir.resolve("rest-api-spec/src/main/resources").resolve(RELATIVE_TEST_PATH))
+                    );
+                task.xpackConfig = xpackCompatTest;
+                project.getDependencies()
+                    .add(
+                        task.xpackConfig.getName(),
+                        project.files(checkoutDir.resolve("x-pack/plugin/src/test/resources").resolve(RELATIVE_TEST_PATH))
+                    );
+                task.additionalConfig = additionalCompatTest;
+                project.getDependencies()
+                    .add(
+                        task.additionalConfig.getName(),
+                        project.files(
+                            getCompatProjectPath(project, checkoutDir).resolve("src/yamlRestTest/resources").resolve(RELATIVE_TEST_PATH)
+                        )
+                    );
+                task.dependsOn(task.coreConfig);
+                task.dependsOn(task.xpackConfig);
+                task.dependsOn(task.additionalConfig);
+                task.dependsOn(":distribution:bwc:minor:checkoutBwcBranch");
+                task.dependsOn(copyCompatYamlSpecTask);
+            });
+
+        // setup the yamlRestTest task
+        Provider<RestIntegTestTask> yamlRestCompatTestTask = RestTestUtil.registerTask(project, yamlCompatTestSourceSet);
+        project.getTasks().withType(RestIntegTestTask.class).named(SOURCE_SET_NAME).configure(testTask -> {
+            // Use test runner and classpath from "normal" yaml source set
+            testTask.setTestClassesDirs(yamlTestSourceSet.getOutput().getClassesDirs());
+            testTask.setClasspath(
+                yamlTestSourceSet.getRuntimeClasspath()
+                    // remove the "normal" api and tests
+                    .minus(project.files(yamlTestSourceSet.getOutput().getResourcesDir()))
+                    // add any additional classes/resources from the compatible source set
+                    // the api and tests are copied to the compatible source set
+                    .plus(yamlCompatTestSourceSet.getRuntimeClasspath())
+            );
+            // run compatibility tests after "normal" tests
+            testTask.mustRunAfter(project.getTasks().named(YamlRestTestPlugin.SOURCE_SET_NAME));
+            testTask.dependsOn(copyCompatYamlTestTask);
+        });
+
+        // setup the dependencies
+        setupDependencies(project, yamlCompatTestSourceSet);
+
+        // setup IDE
+        GradleUtils.setupIdeForTestSourceSet(project, yamlCompatTestSourceSet);
+
+        // wire this task into check
+        project.getTasks().named(JavaBasePlugin.CHECK_TASK_NAME).configure(check -> check.dependsOn(yamlRestCompatTestTask));
+    }
+
+    // TODO: implement custom extension that allows us move around of the projects between major versions and still find them
+    private Path getCompatProjectPath(Project project, Path checkoutDir) {
+        return checkoutDir.resolve(project.getPath().replaceFirst(":", "").replace(":", File.separator));
+    }
+}