Use services for archive and file operations in tasks (#62968)

Referencing a project instance during task execution is discouraged by
Gradle and should be avoided. E.g. It is incompatible with Gradles
incubating configuration cache. Instead there are services available to handle
archive and filesystem operations in task actions.

Brings us one step closer to #57918

buildSrc/src/main/groovy/org/elasticsearch/gradle/AntTask.groovy

@@ -25,9 +25,11 @@ import org.apache.tools.ant.DefaultLogger
 import org.apache.tools.ant.Project
 import org.gradle.api.DefaultTask
 import org.gradle.api.GradleException
+import org.gradle.api.file.FileSystemOperations
 import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.TaskAction
 
+import javax.inject.Inject
 import java.nio.charset.Charset
 
 /**
@@ -43,6 +45,11 @@ public abstract class AntTask extends DefaultTask {
      */
     public final ByteArrayOutputStream outputBuffer = new ByteArrayOutputStream()
 
+    @Inject
+    protected FileSystemOperations getFileSystemOperations() {
+        throw new UnsupportedOperationException();
+    }
+
     @TaskAction
     final void executeTask() {
         AntBuilder ant = new AntBuilder()

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/LicenseHeadersTask.groovy

@@ -35,7 +35,7 @@ import java.nio.file.Files
  * <p>
  * This is a port of the apache lucene check
  */
-public class LicenseHeadersTask extends AntTask {
+class LicenseHeadersTask extends AntTask {
 
     @OutputFile
     File reportFile = new File(project.buildDir, 'reports/licenseHeaders/rat.log')

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy

@@ -28,7 +28,7 @@ class PrecommitTasks {
 
     /** Adds a precommit task, which depends on non-test verification tasks. */
 
-    public static void create(Project project, boolean includeDependencyLicenses) {
+    static void create(Project project, boolean includeDependencyLicenses) {
 
         project.pluginManager.apply(CheckstylePrecommitPlugin)
         project.pluginManager.apply(ForbiddenApisPrecommitPlugin)

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/AntFixture.groovy

@@ -96,7 +96,10 @@ class AntFixture extends AntTask implements Fixture {
 
     @Override
     protected void runAnt(AntBuilder ant) {
-        project.delete(baseDir) // reset everything
+        // reset everything
+        getFileSystemOperations().delete {
+            it.delete(baseDir)
+        }
         cwd.mkdirs()
         final String realExecutable
         final List<Object> realArgs = new ArrayList<>()
@@ -242,7 +245,9 @@ class AntFixture extends AntTask implements Fixture {
                 args('-9', pid)
             }
             doLast {
-                project.delete(fixture.pidFile)
+                getFileSystemOperations().delete {
+                    it.delete(fixture.pidFile)
+                }
             }
 
         }

buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchTestBasePlugin.java

@@ -37,6 +37,7 @@ import org.gradle.api.tasks.testing.Test;
 import java.io.File;
 import java.util.Map;
 
+import static org.elasticsearch.gradle.util.FileUtils.mkdirs;
 import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure;
 
 /**
@@ -83,10 +84,10 @@ public class ElasticsearchTestBasePlugin implements Plugin<Project> {
             test.doFirst(new Action<>() {
                 @Override
                 public void execute(Task t) {
-                    project.mkdir(testOutputDir);
-                    project.mkdir(heapdumpDir);
-                    project.mkdir(test.getWorkingDir());
-                    project.mkdir(test.getWorkingDir().toPath().resolve("temp"));
+                    mkdirs(testOutputDir);
+                    mkdirs(heapdumpDir);
+                    mkdirs(test.getWorkingDir());
+                    mkdirs(test.getWorkingDir().toPath().resolve("temp").toFile());
 
                     // TODO remove once jvm.options are added to test system properties
                     test.systemProperty("java.locale.providers", "SPI,COMPAT");

buildSrc/src/main/java/org/elasticsearch/gradle/FileSystemOperationsAware.java

@@ -0,0 +1,30 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle;
+
+import org.gradle.api.tasks.WorkResult;
+
+/**
+ * An interface for tasks that support basic file operations.
+ * Methods will be added as needed.
+ */
+public interface FileSystemOperationsAware {
+    WorkResult delete(Object... objects);
+}

buildSrc/src/main/java/org/elasticsearch/gradle/LoggedExec.java

@@ -22,15 +22,18 @@ import org.gradle.api.Action;
 import org.gradle.api.GradleException;
 import org.gradle.api.Project;
 import org.gradle.api.Task;
+import org.gradle.api.file.FileSystemOperations;
 import org.gradle.api.logging.Logger;
 import org.gradle.api.logging.Logging;
 import org.gradle.api.tasks.Exec;
+import org.gradle.api.tasks.WorkResult;
 import org.gradle.process.BaseExecSpec;
 import org.gradle.process.ExecOperations;
 import org.gradle.process.ExecResult;
 import org.gradle.process.ExecSpec;
 import org.gradle.process.JavaExecSpec;
 
+import javax.inject.Inject;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.IOException;
@@ -47,13 +50,15 @@ import java.util.regex.Pattern;
  * A wrapper around gradle's Exec task to capture output and log on error.
  */
 @SuppressWarnings("unchecked")
-public class LoggedExec extends Exec {
+public class LoggedExec extends Exec implements FileSystemOperationsAware {
 
     private static final Logger LOGGER = Logging.getLogger(LoggedExec.class);
     private Consumer<Logger> outputLogger;
+    private FileSystemOperations fileSystemOperations;
 
-    public LoggedExec() {
-
+    @Inject
+    public LoggedExec(FileSystemOperations fileSystemOperations) {
+        this.fileSystemOperations = fileSystemOperations;
         if (getLogger().isInfoEnabled() == false) {
             setIgnoreExitValue(true);
             setSpoolOutput(false);
@@ -154,4 +159,9 @@ public class LoggedExec extends Exec {
             throw e;
         }
     }
+
+    @Override
+    public WorkResult delete(Object... objects) {
+        return fileSystemOperations.delete(d -> d.delete(objects));
+    }
 }

buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java

@@ -24,7 +24,9 @@ import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.DefaultTask;
 import org.gradle.api.Project;
 import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.file.ArchiveOperations;
 import org.gradle.api.file.ConfigurableFileCollection;
+import org.gradle.api.file.FileSystemOperations;
 import org.gradle.api.file.FileTree;
 import org.gradle.api.plugins.JavaPluginConvention;
 import org.gradle.api.provider.ListProperty;
@@ -47,6 +49,8 @@ import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.gradle.util.GradleUtils.getProjectPathFromTask;
+
 /**
  * Copies the files needed for the Rest YAML specs to the current projects test resources output directory.
  * This is intended to be be used from {@link RestResourcesPlugin} since the plugin wires up the needed
@@ -76,6 +80,16 @@ public class CopyRestApiTask extends DefaultTask {
         throw new UnsupportedOperationException();
     }
 
+    @Inject
+    protected FileSystemOperations getFileSystemOperations() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Inject
+    protected ArchiveOperations getArchiveOperations() {
+        throw new UnsupportedOperationException();
+    }
+
     @Input
     public ListProperty<String> getIncludeCore() {
         return includeCore;
@@ -137,11 +151,11 @@ public class CopyRestApiTask extends DefaultTask {
 
     @TaskAction
     void copy() {
-        Project project = getProject();
         // always copy the core specs if the task executes
+        String projectPath = getProjectPathFromTask(getPath());
         if (BuildParams.isInternal()) {
-            getLogger().debug("Rest specs for project [{}] will be copied to the test resources.", project.getPath());
-            project.copy(c -> {
+            getLogger().debug("Rest specs for project [{}] will be copied to the test resources.", projectPath);
+            getFileSystemOperations().copy(c -> {
                 c.from(coreConfig.getAsFileTree());
                 c.into(getOutputDir());
                 c.include(corePatternSet.getIncludes());
@@ -149,11 +163,11 @@ public class CopyRestApiTask extends DefaultTask {
         } else {
             getLogger().debug(
                 "Rest specs for project [{}] will be copied to the test resources from the published jar (version: [{}]).",
-                project.getPath(),
+                projectPath,
                 VersionProperties.getElasticsearch()
             );
-            project.copy(c -> {
-                c.from(project.zipTree(coreConfig.getSingleFile()));
+            getFileSystemOperations().copy(c -> {
+                c.from(getArchiveOperations().zipTree(coreConfig.getSingleFile()));
                 // this ends up as the same dir as outputDir
                 c.into(Objects.requireNonNull(getSourceSet().orElseThrow().getOutput().getResourcesDir()));
                 if (includeCore.get().isEmpty()) {
@@ -167,8 +181,8 @@ public class CopyRestApiTask extends DefaultTask {
         }
         // only copy x-pack specs if explicitly instructed
         if (includeXpack.get().isEmpty() == false) {
-            getLogger().debug("X-pack rest specs for project [{}] will be copied to the test resources.", project.getPath());
-            project.copy(c -> {
+            getLogger().debug("X-pack rest specs for project [{}] will be copied to the test resources.", projectPath);
+            getFileSystemOperations().copy(c -> {
                 c.from(xpackConfig.getSingleFile());
                 c.into(getOutputDir());
                 c.include(xpackPatternSet.getIncludes());
@@ -177,7 +191,7 @@ public class CopyRestApiTask extends DefaultTask {
         // TODO: once https://github.com/elastic/elasticsearch/pull/62968 lands ensure that this uses `getFileSystemOperations()`
         // copy any additional config
         if (additionalConfig != null) {
-            project.copy(c -> {
+            getFileSystemOperations().copy(c -> {
                 c.from(additionalConfig.getAsFileTree());
                 c.into(getOutputDir());
             });

buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java

@@ -24,7 +24,9 @@ import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.DefaultTask;
 import org.gradle.api.Project;
 import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.file.ArchiveOperations;
 import org.gradle.api.file.ConfigurableFileCollection;
+import org.gradle.api.file.FileSystemOperations;
 import org.gradle.api.file.FileTree;
 import org.gradle.api.plugins.JavaPluginConvention;
 import org.gradle.api.provider.ListProperty;
@@ -44,6 +46,8 @@ import java.util.Objects;
 import java.util.Optional;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.gradle.util.GradleUtils.getProjectPathFromTask;
+
 /**
  * Copies the Rest YAML test to the current projects test resources output directory.
  * This is intended to be be used from {@link RestResourcesPlugin} since the plugin wires up the needed
@@ -73,6 +77,16 @@ public class CopyRestTestsTask extends DefaultTask {
         throw new UnsupportedOperationException();
     }
 
+    @Inject
+    protected FileSystemOperations getFileSystemOperations() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Inject
+    protected ArchiveOperations getArchiveOperations() {
+        throw new UnsupportedOperationException();
+    }
+
     @Input
     public ListProperty<String> getIncludeCore() {
         return includeCore;
@@ -127,25 +141,24 @@ public class CopyRestTestsTask extends DefaultTask {
 
     @TaskAction
     void copy() {
-        Project project = getProject();
+        String projectPath = getProjectPathFromTask(getPath());
         // only copy core tests if explicitly instructed
         if (includeCore.get().isEmpty() == false) {
             if (BuildParams.isInternal()) {
-                getLogger().debug("Rest tests for project [{}] will be copied to the test resources.", project.getPath());
-                project.copy(c -> {
+                getLogger().debug("Rest tests for project [{}] will be copied to the test resources.", projectPath);
+                getFileSystemOperations().copy(c -> {
                     c.from(coreConfig.getAsFileTree());
                     c.into(getOutputDir());
                     c.include(corePatternSet.getIncludes());
                 });
-
             } else {
                 getLogger().debug(
                     "Rest tests for project [{}] will be copied to the test resources from the published jar (version: [{}]).",
-                    project.getPath(),
+                    projectPath,
                     VersionProperties.getElasticsearch()
                 );
-                project.copy(c -> {
-                    c.from(project.zipTree(coreConfig.getSingleFile()));
+                getFileSystemOperations().copy(c -> {
+                    c.from(getArchiveOperations().zipTree(coreConfig.getSingleFile()));
                     // this ends up as the same dir as outputDir
                     c.into(Objects.requireNonNull(getSourceSet().orElseThrow().getOutput().getResourcesDir()));
                     c.include(
@@ -156,17 +169,16 @@ public class CopyRestTestsTask extends DefaultTask {
         }
         // only copy x-pack tests if explicitly instructed
         if (includeXpack.get().isEmpty() == false) {
-            getLogger().debug("X-pack rest tests for project [{}] will be copied to the test resources.", project.getPath());
-            project.copy(c -> {
+            getLogger().debug("X-pack rest tests for project [{}] will be copied to the test resources.", projectPath);
+            getFileSystemOperations().copy(c -> {
                 c.from(xpackConfig.getAsFileTree());
                 c.into(getOutputDir());
                 c.include(xpackPatternSet.getIncludes());
             });
         }
-        // TODO: once https://github.com/elastic/elasticsearch/pull/62968 lands ensure that this uses `getFileSystemOperations()`
         // copy any additional config
         if (additionalConfig != null) {
-            project.copy(c -> {
+            getFileSystemOperations().copy(c -> {
                 c.from(additionalConfig.getAsFileTree());
                 c.into(getOutputDir());
             });

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java

@@ -25,6 +25,8 @@ import org.elasticsearch.gradle.http.WaitForHttpResource;
 import org.gradle.api.Named;
 import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Project;
+import org.gradle.api.file.ArchiveOperations;
+import org.gradle.api.file.FileSystemOperations;
 import org.gradle.api.file.RegularFile;
 import org.gradle.api.logging.Logger;
 import org.gradle.api.logging.Logging;
@@ -63,16 +65,30 @@ public class ElasticsearchCluster implements TestClusterConfiguration, Named {
     private final LinkedHashMap<String, Predicate<TestClusterConfiguration>> waitConditions = new LinkedHashMap<>();
     private final Project project;
     private final ReaperService reaper;
+    private final FileSystemOperations fileSystemOperations;
+    private final ArchiveOperations archiveOperations;
     private int nodeIndex = 0;
 
-    public ElasticsearchCluster(String path, String clusterName, Project project, ReaperService reaper, File workingDirBase) {
+    public ElasticsearchCluster(
+        String path,
+        String clusterName,
+        Project project,
+        ReaperService reaper,
+        FileSystemOperations fileSystemOperations,
+        ArchiveOperations archiveOperations,
+        File workingDirBase
+    ) {
         this.path = path;
         this.clusterName = clusterName;
         this.project = project;
         this.reaper = reaper;
+        this.fileSystemOperations = fileSystemOperations;
+        this.archiveOperations = archiveOperations;
         this.workingDirBase = workingDirBase;
         this.nodes = project.container(ElasticsearchNode.class);
-        this.nodes.add(new ElasticsearchNode(path, clusterName + "-0", project, reaper, workingDirBase));
+        this.nodes.add(
+            new ElasticsearchNode(path, clusterName + "-0", project, reaper, fileSystemOperations, archiveOperations, workingDirBase)
+        );
         // configure the cluster name eagerly so nodes know about it
         this.nodes.all((node) -> node.defaultConfig.put("cluster.name", safeName(clusterName)));
 
@@ -93,7 +109,9 @@ public class ElasticsearchCluster implements TestClusterConfiguration, Named {
         }
 
         for (int i = nodes.size(); i < numberOfNodes; i++) {
-            this.nodes.add(new ElasticsearchNode(path, clusterName + "-" + i, project, reaper, workingDirBase));
+            this.nodes.add(
+                new ElasticsearchNode(path, clusterName + "-" + i, project, reaper, fileSystemOperations, archiveOperations, workingDirBase)
+            );
         }
     }
 

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -38,6 +38,8 @@ import org.gradle.api.Named;
 import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Project;
 import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.file.ArchiveOperations;
+import org.gradle.api.file.FileSystemOperations;
 import org.gradle.api.file.FileTree;
 import org.gradle.api.file.RegularFile;
 import org.gradle.api.logging.Logger;
@@ -121,6 +123,9 @@ public class ElasticsearchNode implements TestClusterConfiguration {
     private final String name;
     private final Project project;
     private final ReaperService reaper;
+    private final FileSystemOperations fileSystemOperations;
+    private final ArchiveOperations archiveOperations;
+
     private final AtomicBoolean configurationFrozen = new AtomicBoolean(false);
     private final Path workingDir;
 
@@ -161,11 +166,21 @@ public class ElasticsearchNode implements TestClusterConfiguration {
     private Path confPathData;
     private String keystorePassword = "";
 
-    ElasticsearchNode(String path, String name, Project project, ReaperService reaper, File workingDirBase) {
+    ElasticsearchNode(
+        String path,
+        String name,
+        Project project,
+        ReaperService reaper,
+        FileSystemOperations fileSystemOperations,
+        ArchiveOperations archiveOperations,
+        File workingDirBase
+    ) {
         this.path = path;
         this.name = name;
         this.project = project;
         this.reaper = reaper;
+        this.fileSystemOperations = fileSystemOperations;
+        this.archiveOperations = archiveOperations;
         workingDir = workingDirBase.toPath().resolve(safeName(name)).toAbsolutePath();
         confPathRepo = workingDir.resolve("repo");
         configFile = workingDir.resolve("config/elasticsearch.yml");
@@ -433,7 +448,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
                 logToProcessStdout("Configuring working directory: " + workingDir);
                 // make sure we always start fresh
                 if (Files.exists(workingDir)) {
-                    project.delete(workingDir);
+                    fileSystemOperations.delete(d -> d.delete(workingDir));
                 }
                 isWorkingDirConfigured = true;
             }
@@ -605,9 +620,9 @@ public class ElasticsearchNode implements TestClusterConfiguration {
                     .resolve(module.get().getName().replace(".zip", "").replace("-" + getVersion(), "").replace("-SNAPSHOT", ""));
                 // only install modules that are not already bundled with the integ-test distribution
                 if (Files.exists(destination) == false) {
-                    project.copy(spec -> {
+                    fileSystemOperations.copy(spec -> {
                         if (module.get().getName().toLowerCase().endsWith(".zip")) {
-                            spec.from(project.zipTree(module));
+                            spec.from(archiveOperations.zipTree(module));
                         } else if (module.get().isDirectory()) {
                             spec.from(module);
                         } else {
@@ -1005,7 +1020,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
 
     private void createWorkingDir() throws IOException {
         // Start configuration from scratch in case of a restart
-        project.delete(configFile.getParent());
+        fileSystemOperations.delete(d -> d.delete(configFile.getParent()));
         Files.createDirectories(configFile.getParent());
         Files.createDirectories(confPathRepo);
         Files.createDirectories(confPathData);
@@ -1251,7 +1266,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
             .filter(File::exists)
             // TODO: We may be able to simplify this with Gradle 5.6
             // https://docs.gradle.org/nightly/release-notes.html#improved-handling-of-zip-archives-on-classpaths
-            .map(zipFile -> project.zipTree(zipFile).matching(filter))
+            .map(zipFile -> archiveOperations.zipTree(zipFile).matching(filter))
             .flatMap(tree -> tree.getFiles().stream())
             .sorted(Comparator.comparing(File::getName))
             .collect(Collectors.toList());

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/StandaloneRestIntegTestTask.java

@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.gradle.testclusters;
 
+import org.elasticsearch.gradle.FileSystemOperationsAware;
 import org.elasticsearch.gradle.test.Fixture;
 import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.Task;
@@ -26,6 +27,7 @@ import org.gradle.api.services.internal.BuildServiceRegistryInternal;
 import org.gradle.api.tasks.CacheableTask;
 import org.gradle.api.tasks.Internal;
 import org.gradle.api.tasks.Nested;
+import org.gradle.api.tasks.WorkResult;
 import org.gradle.api.tasks.testing.Test;
 import org.gradle.internal.resources.ResourceLock;
 import org.gradle.internal.resources.SharedResource;
@@ -44,7 +46,7 @@ import static org.elasticsearch.gradle.testclusters.TestClustersPlugin.THROTTLE_
  * {@link Nested} inputs.
  */
 @CacheableTask
-public class StandaloneRestIntegTestTask extends Test implements TestClustersAware {
+public class StandaloneRestIntegTestTask extends Test implements TestClustersAware, FileSystemOperationsAware {
 
     private Collection<ElasticsearchCluster> clusters = new HashSet<>();
 
@@ -113,4 +115,8 @@ public class StandaloneRestIntegTestTask extends Test implements TestClustersAwa
             }
         }
     }
+
+    public WorkResult delete(Object... objects) {
+        return getFileSystemOperations().delete(d -> d.delete(objects));
+    }
 }

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java

@@ -31,12 +31,15 @@ import org.gradle.api.Project;
 import org.gradle.api.Task;
 import org.gradle.api.execution.TaskActionListener;
 import org.gradle.api.execution.TaskExecutionListener;
+import org.gradle.api.file.ArchiveOperations;
+import org.gradle.api.file.FileSystemOperations;
 import org.gradle.api.invocation.Gradle;
 import org.gradle.api.logging.Logger;
 import org.gradle.api.logging.Logging;
 import org.gradle.api.provider.Provider;
 import org.gradle.api.tasks.TaskState;
 
+import javax.inject.Inject;
 import java.io.File;
 
 import static org.elasticsearch.gradle.util.GradleUtils.noop;
@@ -50,6 +53,16 @@ public class TestClustersPlugin implements Plugin<Project> {
     private static final String REGISTRY_SERVICE_NAME = "testClustersRegistry";
     private static final Logger logger = Logging.getLogger(TestClustersPlugin.class);
 
+    @Inject
+    protected FileSystemOperations getFileSystemOperations() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Inject
+    protected ArchiveOperations getArchiveOperations() {
+        throw new UnsupportedOperationException();
+    }
+
     @Override
     public void apply(Project project) {
         project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class);
@@ -89,7 +102,15 @@ public class TestClustersPlugin implements Plugin<Project> {
         // Create an extensions that allows describing clusters
         NamedDomainObjectContainer<ElasticsearchCluster> container = project.container(
             ElasticsearchCluster.class,
-            name -> new ElasticsearchCluster(project.getPath(), name, project, reaper, new File(project.getBuildDir(), "testclusters"))
+            name -> new ElasticsearchCluster(
+                project.getPath(),
+                name,
+                project,
+                reaper,
+                getFileSystemOperations(),
+                getArchiveOperations(),
+                new File(project.getBuildDir(), "testclusters")
+            )
         );
         project.getExtensions().add(EXTENSION_NAME, container);
         return container;

buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixturesPlugin.java

@@ -35,6 +35,7 @@ import org.gradle.api.DefaultTask;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.Task;
+import org.gradle.api.file.FileSystemOperations;
 import org.gradle.api.logging.Logger;
 import org.gradle.api.logging.Logging;
 import org.gradle.api.plugins.BasePlugin;
@@ -44,6 +45,7 @@ import org.gradle.api.tasks.TaskContainer;
 import org.gradle.api.tasks.TaskProvider;
 import org.gradle.api.tasks.testing.Test;
 
+import javax.inject.Inject;
 import java.io.File;
 import java.io.IOException;
 import java.io.UncheckedIOException;
@@ -57,6 +59,11 @@ public class TestFixturesPlugin implements Plugin<Project> {
     private static final String DOCKER_COMPOSE_THROTTLE = "dockerComposeThrottle";
     static final String DOCKER_COMPOSE_YML = "docker-compose.yml";
 
+    @Inject
+    protected FileSystemOperations getFileSystemOperations() {
+        throw new UnsupportedOperationException();
+    }
+
     @Override
     public void apply(Project project) {
         project.getRootProject().getPluginManager().apply(DockerSupportPlugin.class);
@@ -122,7 +129,7 @@ public class TestFixturesPlugin implements Plugin<Project> {
                 t.mustRunAfter(preProcessFixture);
             });
             tasks.named("composePull").configure(t -> t.mustRunAfter(preProcessFixture));
-            tasks.named("composeDown").configure(t -> t.doLast(t2 -> project.delete(testfixturesDir)));
+            tasks.named("composeDown").configure(t -> t.doLast(t2 -> getFileSystemOperations().delete(d -> d.delete(testfixturesDir))));
         } else {
             project.afterEvaluate(spec -> {
                 if (extension.fixtures.isEmpty()) {

buildSrc/src/main/java/org/elasticsearch/gradle/util/FileUtils.java

@@ -0,0 +1,78 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle.util;
+
+import org.gradle.api.UncheckedIOException;
+
+import java.io.File;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+
+public final class FileUtils {
+
+    /**
+     * Like {@link java.io.File#mkdirs()}, except throws an informative error if a dir cannot be created.
+     *
+     * @param dir The dir to create, including any non existent parent dirs.
+     */
+    public static void mkdirs(File dir) {
+        dir = dir.getAbsoluteFile();
+        if (dir.isDirectory()) {
+            return;
+        }
+
+        if (dir.exists() && !dir.isDirectory()) {
+            throw new UncheckedIOException(String.format("Cannot create directory '%s' as it already exists, but is not a directory", dir));
+        }
+
+        List<File> toCreate = new LinkedList<File>();
+        File parent = dir.getParentFile();
+        while (!parent.exists()) {
+            toCreate.add(parent);
+            parent = parent.getParentFile();
+        }
+        Collections.reverse(toCreate);
+        for (File parentDirToCreate : toCreate) {
+            if (parentDirToCreate.isDirectory()) {
+                continue;
+            }
+            File parentDirToCreateParent = parentDirToCreate.getParentFile();
+            if (!parentDirToCreateParent.isDirectory()) {
+                throw new UncheckedIOException(
+                    String.format(
+                        "Cannot create parent directory '%s' when creating directory '%s' as '%s' is not a directory",
+                        parentDirToCreate,
+                        dir,
+                        parentDirToCreateParent
+                    )
+                );
+            }
+            if (!parentDirToCreate.mkdir() && !parentDirToCreate.isDirectory()) {
+                throw new UncheckedIOException(
+                    String.format("Failed to create parent directory '%s' when creating directory '%s'", parentDirToCreate, dir)
+                );
+            }
+        }
+        if (!dir.mkdir() && !dir.isDirectory()) {
+            throw new UncheckedIOException(String.format("Failed to create directory '%s'", dir));
+        }
+    }
+}

buildSrc/src/main/java/org/elasticsearch/gradle/util/GradleUtils.java

@@ -112,7 +112,7 @@ public abstract class GradleUtils {
 
     /**
      * Add a source set and task of the same name that runs tests.
-     *
+     * <p>
      * IDEs are also configured if setup, and the test task is added to check. The new test source
      * set extends from the normal test source set to allow sharing of utilities.
      *
@@ -218,4 +218,13 @@ public abstract class GradleUtils {
         depConfig.put("configuration", projectConfig);
         return project.getDependencies().project(depConfig);
     }
+
+    /**
+     * To calculate the project path from a task path without relying on Task#getProject() which is discouraged during
+     * task execution time.
+     */
+    public static String getProjectPathFromTask(String taskPath) {
+        int lastDelimiterIndex = taskPath.lastIndexOf(":");
+        return lastDelimiterIndex == 0 ? ":" : taskPath.substring(0, lastDelimiterIndex);
+    }
 }

distribution/packages/build.gradle

@@ -451,7 +451,7 @@ subprojects {
     tasks.register('checkExtraction', LoggedExec) {
       dependsOn buildDist
       doFirst {
-        project.delete(extractionDir)
+        delete(extractionDir)
         extractionDir.mkdirs()
       }
     }

plugins/discovery-azure-classic/build.gradle

@@ -74,7 +74,7 @@ File keystore = new File(project.buildDir, 'keystore/test-node.jks')
 // generate the keystore
 TaskProvider createKey = tasks.register("createKey", LoggedExec) {
   doFirst {
-    project.delete(keystore.parentFile)
+    delete(keystore.parentFile)
     keystore.parentFile.mkdirs()
   }
   outputs.file(keystore).withPropertyName('keystoreFile')

qa/full-cluster-restart/build.gradle

@@ -43,7 +43,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.indexCompatible) {
     useCluster testClusters."${baseName}"
     mustRunAfter(precommit)
     doFirst {
-      project.delete("${buildDir}/cluster/shared/repo/${baseName}")
+      delete("${buildDir}/cluster/shared/repo/${baseName}")
     }
 
     systemProperty 'tests.is_old_cluster', 'true'

qa/mixed-cluster/build.gradle

@@ -56,7 +56,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) {
     useCluster testClusters."${baseName}"
     mustRunAfter(precommit)
     doFirst {
-      project.delete("${buildDir}/cluster/shared/repo/${baseName}")
+      delete("${buildDir}/cluster/shared/repo/${baseName}")
       // Getting the endpoints causes a wait for the cluster
       println "Test cluster endpoints are: ${-> testClusters."${baseName}".allHttpSocketURI.join(",")}"
       println "Upgrading one node to create a mixed cluster"

qa/repository-multi-version/build.gradle

@@ -51,7 +51,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.indexCompatible) {
     useCluster testClusters."${oldClusterName}"
     mustRunAfter(precommit)
     doFirst {
-      project.delete("${buildDir}/cluster/shared/repo/${baseName}")
+      delete("${buildDir}/cluster/shared/repo/${baseName}")
     }
     systemProperty 'tests.rest.suite', 'step1'
   }

qa/rolling-upgrade/build.gradle

@@ -57,7 +57,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) {
     useCluster testClusters."${baseName}"
     mustRunAfter(precommit)
     doFirst {
-      project.delete("${buildDir}/cluster/shared/repo/${baseName}")
+      delete("${buildDir}/cluster/shared/repo/${baseName}")
     }
     systemProperty 'tests.rest.suite', 'old_cluster'
     nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")

x-pack/qa/full-cluster-restart/build.gradle

@@ -73,7 +73,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.indexCompatible) {
     useCluster testClusters."${baseName}"
     dependsOn copyTestNodeKeyMaterial
     doFirst {
-      project.delete("${buildDir}/cluster/shared/repo/${baseName}")
+      delete("${buildDir}/cluster/shared/repo/${baseName}")
     }
     systemProperty 'tests.is_old_cluster', 'true'
     exclude 'org/elasticsearch/upgrades/FullClusterRestartIT.class'