Improve testcluster distribution artifact handling (#38933)

This commit moves validation logic for ensuring our testclusters
configuration doesn't contain unexpected artifacts into the plugin
itself. This change allows us to remove the custom copy task
implementation altogether.

Additionally, the error message has been improved to display component
ids in addition to the artifacts to make it easier to figure out what
actual dependency is at fault.

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/SyncTestClustersConfiguration.java

@@ -1,77 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.gradle.testclusters;
-
-import org.gradle.api.DefaultTask;
-import org.gradle.api.Project;
-import org.gradle.api.file.FileCollection;
-import org.gradle.api.tasks.InputFiles;
-import org.gradle.api.tasks.OutputDirectory;
-import org.gradle.api.tasks.TaskAction;
-
-import java.io.File;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-public class SyncTestClustersConfiguration extends DefaultTask {
-
-    @InputFiles
-    public FileCollection getDependencies() {
-        Set<File> nonZip = getProject().getConfigurations()
-            .getByName(TestClustersPlugin.HELPER_CONFIGURATION_NAME)
-            .getFiles()
-            .stream()
-            .filter(file -> file.getName().endsWith(".zip") == false)
-            .collect(Collectors.toSet());
-        if(nonZip.isEmpty() == false) {
-            throw new IllegalStateException("Expected only zip files in configuration : " +
-                TestClustersPlugin.HELPER_CONFIGURATION_NAME + " but found " +
-                nonZip
-            );
-        }
-        return getProject().files(
-            getProject().getConfigurations()
-                .getByName(TestClustersPlugin.HELPER_CONFIGURATION_NAME)
-                .getFiles()
-        );
-    }
-
-    @OutputDirectory
-    public File getOutputDir() {
-        return getTestClustersConfigurationExtractDir(getProject());
-    }
-
-    @TaskAction
-    public void doExtract() {
-        File outputDir = getOutputDir();
-        getProject().delete(outputDir);
-        outputDir.mkdirs();
-        getDependencies().forEach(dep ->
-            getProject().copy(spec -> {
-                spec.from(getProject().zipTree(dep));
-                spec.into(new File(outputDir, "zip"));
-            })
-        );
-    }
-
-    static File getTestClustersConfigurationExtractDir(Project project) {
-        return new File(TestClustersPlugin.getTestClustersBuildDir(project), "extract");
-    }
-
-}

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java

@@ -25,11 +25,14 @@ import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.Task;
 import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.artifacts.component.ComponentArtifactIdentifier;
 import org.gradle.api.execution.TaskActionListener;
 import org.gradle.api.execution.TaskExecutionListener;
+import org.gradle.api.file.FileTree;
 import org.gradle.api.logging.Logger;
 import org.gradle.api.logging.Logging;
 import org.gradle.api.plugins.ExtraPropertiesExtension;
+import org.gradle.api.tasks.Sync;
 import org.gradle.api.tasks.TaskState;
 
 import java.io.File;
@@ -39,6 +42,7 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -87,6 +91,20 @@ public class TestClustersPlugin implements Plugin<Project> {
                 "Internal helper configuration used by cluster configuration to download " +
                     "ES distributions and plugins."
             );
+            helperConfiguration.getIncoming().afterResolve(resolvableDependencies -> {
+                Set<ComponentArtifactIdentifier> nonZipComponents = resolvableDependencies.getArtifacts()
+                    .getArtifacts()
+                    .stream()
+                    .filter(artifact -> artifact.getFile().getName().endsWith(".zip") == false)
+                    .map(artifact -> artifact.getId())
+                    .collect(Collectors.toSet());
+
+                if(nonZipComponents.isEmpty() == false) {
+                    throw new IllegalStateException("Dependencies with non-zip artifacts found in configuration '" +
+                        TestClustersPlugin.HELPER_CONFIGURATION_NAME + "': " + nonZipComponents
+                    );
+                }
+            });
 
             // When running in the Daemon it's possible for this to hold references to past
             usedClusters.clear();
@@ -98,7 +116,15 @@ public class TestClustersPlugin implements Plugin<Project> {
             // the clusters will look for artifacts there based on the naming conventions.
             // Tasks that use a cluster will add this as a dependency automatically so it's guaranteed to run early in
             // the build.
-            rootProject.getTasks().create(SYNC_ARTIFACTS_TASK_NAME, SyncTestClustersConfiguration.class);
+            rootProject.getTasks().create(SYNC_ARTIFACTS_TASK_NAME, Sync.class, sync -> {
+                sync.from((Callable<List<FileTree>>) () ->
+                    helperConfiguration.getFiles()
+                        .stream()
+                        .map(project::zipTree)
+                        .collect(Collectors.toList())
+                );
+                sync.into(new File(getTestClustersConfigurationExtractDir(project), "zip"));
+            });
 
             // When we know what tasks will run, we claim the clusters of those task to differentiate between clusters
             // that are defined in the build script and the ones that will actually be used in this invocation of gradle
@@ -129,7 +155,7 @@ public class TestClustersPlugin implements Plugin<Project> {
                 project.getPath(),
                 name,
                 GradleServicesAdapter.getInstance(project),
-                SyncTestClustersConfiguration.getTestClustersConfigurationExtractDir(project),
+                getTestClustersConfigurationExtractDir(project),
                 new File(project.getBuildDir(), "testclusters")
             )
         );
@@ -249,8 +275,8 @@ public class TestClustersPlugin implements Plugin<Project> {
         );
     }
 
-    static File getTestClustersBuildDir(Project project) {
-        return new File(project.getRootProject().getBuildDir(), "testclusters");
+    static File getTestClustersConfigurationExtractDir(Project project) {
+        return new File(project.getRootProject().getBuildDir(), "testclusters/extract");
     }
 
     /**