Fix/30904 cluster formation part2 (#32877)

Gradle integration for the Cluster formation plugin with ref counting

buildSrc/build.gradle

@@ -23,6 +23,15 @@ plugins {
   id 'groovy'
 }
 
+gradlePlugin {
+  plugins {
+    simplePlugin {
+      id = 'elasticsearch.clusterformation'
+      implementationClass = 'org.elasticsearch.gradle.clusterformation.ClusterformationPlugin'
+    }
+  }
+}
+
 group = 'org.elasticsearch.gradle'
 
 String minimumGradleVersion = file('src/main/resources/minimumGradleVersion').text.trim()

buildSrc/src/main/java/org/elasticsearch/GradleServicesAdapter.java

@@ -0,0 +1,68 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch;
+
+import org.gradle.api.Action;
+import org.gradle.api.Project;
+import org.gradle.api.file.CopySpec;
+import org.gradle.api.file.FileTree;
+import org.gradle.api.tasks.WorkResult;
+import org.gradle.process.ExecResult;
+import org.gradle.process.JavaExecSpec;
+
+import java.io.File;
+
+/**
+ * Facilitate access to Gradle services without a direct dependency on Project.
+ *
+ * In a future release Gradle will offer service injection, this adapter plays that role until that time.
+ * It exposes the service methods that are part of the public API as the classes implementing them are not.
+ * Today service injection is <a href="https://github.com/gradle/gradle/issues/2363">not available</a> for
+ * extensions.
+ *
+ * Everything exposed here must be thread safe. That is the very reason why project is not passed in directly.
+ */
+public class GradleServicesAdapter {
+
+    public final Project project;
+
+    public GradleServicesAdapter(Project project) {
+        this.project = project;
+    }
+
+    public static GradleServicesAdapter getInstance(Project project) {
+        return new GradleServicesAdapter(project);
+    }
+
+    public WorkResult copy(Action<? super CopySpec> action) {
+        return project.copy(action);
+    }
+
+    public WorkResult sync(Action<? super CopySpec> action) {
+        return project.sync(action);
+    }
+
+    public ExecResult javaexec(Action<? super JavaExecSpec> action) {
+        return project.javaexec(action);
+    }
+
+    public FileTree zipTree(File zipPath) {
+        return project.zipTree(zipPath);
+    }
+}

buildSrc/src/main/java/org/elasticsearch/gradle/Distribution.java

@@ -0,0 +1,36 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle;
+
+public enum Distribution {
+
+    INTEG_TEST("integ-test-zip"),
+    ZIP("zip"),
+    ZIP_OSS("zip-oss");
+
+    private final String name;
+
+    Distribution(String name) {
+        this.name = name;
+    }
+
+    public String getName() {
+        return name;
+    }
+}

buildSrc/src/main/java/org/elasticsearch/gradle/clusterformation/ClusterformationPlugin.java

@@ -0,0 +1,110 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.clusterformation;
+
+import groovy.lang.Closure;
+import org.elasticsearch.GradleServicesAdapter;
+import org.gradle.api.NamedDomainObjectContainer;
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.Task;
+import org.gradle.api.execution.TaskActionListener;
+import org.gradle.api.execution.TaskExecutionListener;
+import org.gradle.api.logging.Logger;
+import org.gradle.api.logging.Logging;
+import org.gradle.api.plugins.ExtraPropertiesExtension;
+import org.gradle.api.tasks.TaskState;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class ClusterformationPlugin implements Plugin<Project> {
+
+    public static final String LIST_TASK_NAME = "listElasticSearchClusters";
+    public static final String EXTENSION_NAME = "elasticSearchClusters";
+
+    private final Logger logger =  Logging.getLogger(ClusterformationPlugin.class);
+
+    @Override
+    public void apply(Project project) {
+        NamedDomainObjectContainer<? extends ElasticsearchConfiguration> container = project.container(
+            ElasticsearchNode.class,
+            (name) -> new ElasticsearchNode(name, GradleServicesAdapter.getInstance(project))
+        );
+        project.getExtensions().add(EXTENSION_NAME, container);
+
+        Task listTask = project.getTasks().create(LIST_TASK_NAME);
+        listTask.setGroup("ES cluster formation");
+        listTask.setDescription("Lists all ES clusters configured for this project");
+        listTask.doLast((Task task) ->
+            container.forEach((ElasticsearchConfiguration cluster) ->
+                logger.lifecycle("   * {}: {}", cluster.getName(), cluster.getDistribution())
+            )
+        );
+
+        Map<Task, List<ElasticsearchConfiguration>> taskToCluster = new HashMap<>();
+
+        // register an extension for all current and future tasks, so that any task can declare that it wants to use a
+        // specific cluster.
+        project.getTasks().all((Task task) ->
+            task.getExtensions().findByType(ExtraPropertiesExtension.class)
+            .set(
+                "useCluster",
+                new Closure<Void>(this, this) {
+                    public void doCall(ElasticsearchConfiguration conf) {
+                        taskToCluster.computeIfAbsent(task, k -> new ArrayList<>()).add(conf);
+                    }
+                })
+        );
+
+        project.getGradle().getTaskGraph().whenReady(taskExecutionGraph ->
+            taskExecutionGraph.getAllTasks()
+                .forEach(task ->
+                    taskToCluster.getOrDefault(task, Collections.emptyList()).forEach(ElasticsearchConfiguration::claim)
+                )
+        );
+        project.getGradle().addListener(
+            new TaskActionListener() {
+                @Override
+                public void beforeActions(Task task) {
+                    // we only start the cluster before the actions, so we'll not start it if the task is up-to-date
+                    taskToCluster.getOrDefault(task, new ArrayList<>()).forEach(ElasticsearchConfiguration::start);
+                }
+                @Override
+                public void afterActions(Task task) {}
+            }
+        );
+        project.getGradle().addListener(
+            new TaskExecutionListener() {
+                @Override
+                public void afterExecute(Task task, TaskState state) {
+                    // always un-claim the cluster, even if _this_ task is up-to-date, as others might not have been and caused the
+                    // cluster to start.
+                    taskToCluster.getOrDefault(task, new ArrayList<>()).forEach(ElasticsearchConfiguration::unClaimAndStop);
+                }
+                @Override
+                public void beforeExecute(Task task) {}
+            }
+        );
+    }
+
+}

buildSrc/src/main/java/org/elasticsearch/gradle/clusterformation/ElasticsearchConfiguration.java

@@ -0,0 +1,46 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.clusterformation;
+
+import org.elasticsearch.gradle.Distribution;
+import org.elasticsearch.gradle.Version;
+
+import java.util.concurrent.Future;
+
+public interface ElasticsearchConfiguration {
+    String getName();
+
+    Version getVersion();
+
+    void setVersion(Version version);
+
+    default void setVersion(String version) {
+        setVersion(Version.fromString(version));
+    }
+
+    Distribution getDistribution();
+
+    void setDistribution(Distribution distribution);
+
+    void claim();
+
+    Future<Void> start();
+
+    void unClaimAndStop();
+}

buildSrc/src/main/java/org/elasticsearch/gradle/clusterformation/ElasticsearchNode.java

@@ -0,0 +1,130 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.clusterformation;
+
+import org.elasticsearch.GradleServicesAdapter;
+import org.elasticsearch.gradle.Distribution;
+import org.elasticsearch.gradle.Version;
+import org.gradle.api.logging.Logger;
+import org.gradle.api.logging.Logging;
+
+import java.util.Objects;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class ElasticsearchNode implements ElasticsearchConfiguration {
+
+    private final String name;
+    private final GradleServicesAdapter services;
+    private final AtomicInteger noOfClaims = new AtomicInteger();
+    private final AtomicBoolean started = new AtomicBoolean(false);
+    private final Logger logger = Logging.getLogger(ElasticsearchNode.class);
+
+    private Distribution distribution;
+    private Version version;
+
+    public ElasticsearchNode(String name, GradleServicesAdapter services) {
+        this.name = name;
+        this.services = services;
+    }
+
+    @Override
+    public String getName() {
+        return name;
+    }
+
+    @Override
+    public Version getVersion() {
+        return version;
+    }
+
+    @Override
+    public void setVersion(Version version) {
+        checkNotRunning();
+        this.version = version;
+    }
+
+    @Override
+    public Distribution getDistribution() {
+        return distribution;
+    }
+
+    @Override
+    public void setDistribution(Distribution distribution) {
+        checkNotRunning();
+        this.distribution = distribution;
+    }
+
+    @Override
+    public void claim() {
+        noOfClaims.incrementAndGet();
+    }
+
+    /**
+     * Start the cluster if not running. Does nothing if the cluster is already running.
+     *
+     * @return future of thread running in the background
+     */
+    @Override
+    public Future<Void> start() {
+        if (started.getAndSet(true)) {
+            logger.lifecycle("Already started cluster: {}", name);
+        } else {
+            logger.lifecycle("Starting cluster: {}", name);
+        }
+        return null;
+    }
+
+    /**
+     * Stops a running cluster if it's not claimed. Does nothing otherwise.
+     */
+    @Override
+    public void unClaimAndStop() {
+        int decrementedClaims = noOfClaims.decrementAndGet();
+        if (decrementedClaims > 0) {
+            logger.lifecycle("Not stopping {}, since cluster still has {} claim(s)", name, decrementedClaims);
+            return;
+        }
+        if (started.get() == false) {
+            logger.lifecycle("Asked to unClaimAndStop, but cluster was not running: {}", name);
+            return;
+        }
+        logger.lifecycle("Stopping {}, number of claims is {}", name, decrementedClaims);
+    }
+
+    private void checkNotRunning() {
+        if (started.get()) {
+            throw new IllegalStateException("Configuration can not be altered while running ");
+        }
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        ElasticsearchNode that = (ElasticsearchNode) o;
+        return Objects.equals(name, that.name);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(name);
+    }
+}

buildSrc/src/test/java/org/elasticsearch/gradle/clusterformation/ClusterformationPluginIT.java

@@ -0,0 +1,144 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.clusterformation;
+
+import org.elasticsearch.gradle.test.GradleIntegrationTestCase;
+import org.gradle.testkit.runner.BuildResult;
+import org.gradle.testkit.runner.GradleRunner;
+import org.gradle.testkit.runner.TaskOutcome;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+public class ClusterformationPluginIT extends GradleIntegrationTestCase {
+
+    public void testListClusters() {
+        BuildResult result = GradleRunner.create()
+            .withProjectDir(getProjectDir("clusterformation"))
+            .withArguments("listElasticSearchClusters", "-s")
+            .withPluginClasspath()
+            .build();
+
+        assertEquals(TaskOutcome.SUCCESS, result.task(":listElasticSearchClusters").getOutcome());
+        assertOutputContains(
+            result.getOutput(),
+                "   * myTestCluster:"
+        );
+
+    }
+
+    public void testUseClusterByOne() {
+        BuildResult result = GradleRunner.create()
+            .withProjectDir(getProjectDir("clusterformation"))
+            .withArguments("user1", "-s")
+            .withPluginClasspath()
+            .build();
+
+        assertEquals(TaskOutcome.SUCCESS, result.task(":user1").getOutcome());
+        assertOutputContains(
+            result.getOutput(),
+                "Starting cluster: myTestCluster",
+                "Stopping myTestCluster, number of claims is 0"
+        );
+    }
+
+    public void testUseClusterByOneWithDryRun() {
+        BuildResult result = GradleRunner.create()
+            .withProjectDir(getProjectDir("clusterformation"))
+            .withArguments("user1", "-s", "--dry-run")
+            .withPluginClasspath()
+            .build();
+
+        assertNull(result.task(":user1"));
+        assertOutputDoesNotContain(
+            result.getOutput(),
+            "Starting cluster: myTestCluster",
+            "Stopping myTestCluster, number of claims is 0"
+        );
+    }
+
+    public void testUseClusterByTwo() {
+        BuildResult result = GradleRunner.create()
+            .withProjectDir(getProjectDir("clusterformation"))
+            .withArguments("user1", "user2", "-s")
+            .withPluginClasspath()
+            .build();
+
+        assertEquals(TaskOutcome.SUCCESS, result.task(":user1").getOutcome());
+        assertEquals(TaskOutcome.SUCCESS, result.task(":user2").getOutcome());
+        assertOutputContains(
+            result.getOutput(),
+            "Starting cluster: myTestCluster",
+            "Not stopping myTestCluster, since cluster still has 1 claim(s)",
+            "Stopping myTestCluster, number of claims is 0"
+        );
+    }
+
+    public void testUseClusterByUpToDateTask() {
+        BuildResult result = GradleRunner.create()
+            .withProjectDir(getProjectDir("clusterformation"))
+            .withArguments("upToDate1", "upToDate2", "-s")
+            .withPluginClasspath()
+            .build();
+
+        assertEquals(TaskOutcome.UP_TO_DATE, result.task(":upToDate1").getOutcome());
+        assertEquals(TaskOutcome.UP_TO_DATE, result.task(":upToDate2").getOutcome());
+        assertOutputContains(
+            result.getOutput(),
+            "Not stopping myTestCluster, since cluster still has 1 claim(s)",
+            "cluster was not running: myTestCluster"
+        );
+        assertOutputDoesNotContain(result.getOutput(), "Starting cluster: myTestCluster");
+    }
+
+    public void testUseClusterBySkippedTask() {
+        BuildResult result = GradleRunner.create()
+            .withProjectDir(getProjectDir("clusterformation"))
+            .withArguments("skipped1", "skipped2", "-s")
+            .withPluginClasspath()
+            .build();
+
+        assertEquals(TaskOutcome.SKIPPED, result.task(":skipped1").getOutcome());
+        assertEquals(TaskOutcome.SKIPPED, result.task(":skipped2").getOutcome());
+        assertOutputContains(
+            result.getOutput(),
+            "Not stopping myTestCluster, since cluster still has 1 claim(s)",
+            "cluster was not running: myTestCluster"
+        );
+        assertOutputDoesNotContain(result.getOutput(), "Starting cluster: myTestCluster");
+    }
+
+    public void tetUseClusterBySkippedAndWorkingTask() {
+        BuildResult result = GradleRunner.create()
+            .withProjectDir(getProjectDir("clusterformation"))
+            .withArguments("skipped1", "user1", "-s")
+            .withPluginClasspath()
+            .build();
+
+        assertEquals(TaskOutcome.SKIPPED, result.task(":skipped1").getOutcome());
+        assertEquals(TaskOutcome.SUCCESS, result.task(":user1").getOutcome());
+        assertOutputContains(
+            result.getOutput(),
+            "> Task :user1",
+            "Starting cluster: myTestCluster",
+            "Stopping myTestCluster, number of claims is 0"
+        );
+    }
+
+}

buildSrc/src/testKit/clusterformation/build.gradle

@@ -0,0 +1,41 @@
+plugins {
+    id 'elasticsearch.clusterformation'
+}
+
+elasticSearchClusters {
+    myTestCluster {
+       distribution = 'ZIP'
+    }
+}
+
+task user1 {
+    useCluster elasticSearchClusters.myTestCluster
+    doLast {
+        println "user1 executing"
+    }
+}
+
+task user2 {
+    useCluster elasticSearchClusters.myTestCluster
+    doLast {
+        println "user2 executing"
+    }
+}
+
+task upToDate1 {
+    useCluster elasticSearchClusters.myTestCluster
+}
+
+task upToDate2 {
+    useCluster elasticSearchClusters.myTestCluster
+}
+
+task skipped1 {
+    enabled = false
+    useCluster elasticSearchClusters.myTestCluster
+}
+
+task skipped2 {
+    enabled = false
+    useCluster elasticSearchClusters.myTestCluster
+}