Testclsuters: convert plugins qa projects (#41496)

Add testclusters support for files in keystore and convert qa subprojects within plugins.

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -89,6 +89,7 @@ class PluginBuildPlugin extends BuildPlugin {
                 project.extensions.getByType(PluginPropertiesExtension).extendedPlugins.each { pluginName ->
                     // Auto add dependent modules to the test cluster
                     if (project.findProject(":modules:${pluginName}") != null) {
+                        project.integTest.dependsOn(project.project(":modules:${pluginName}").tasks.bundlePlugin)
                         project.testClusters.integTest.module(
                                 project.file(project.project(":modules:${pluginName}").tasks.bundlePlugin.archiveFile)
                         )

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy

@@ -120,7 +120,6 @@ class RestIntegTestTask extends DefaultTask {
             if (usesTestclusters == true) {
                 ElasticsearchCluster cluster = project.testClusters."${name}"
                 nonInputProperties.systemProperty('tests.rest.cluster', "${-> cluster.allHttpSocketURI.join(",") }")
-                nonInputProperties.systemProperty('tests.config.dir', "${-> cluster.singleNode().getConfigDir() }")
                 nonInputProperties.systemProperty('tests.cluster', "${-> cluster.transportPortURI }")
             } else {
                 // we pass all nodes to the rest cluster to allow the clients to round-robin between them

buildSrc/src/main/java/org/elasticsearch/gradle/FileSupplier.java

@@ -0,0 +1,7 @@
+package org.elasticsearch.gradle;
+
+import java.io.File;
+import java.util.function.Supplier;
+
+public interface FileSupplier extends Supplier<File> {
+}

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java

@@ -20,6 +20,7 @@ package org.elasticsearch.gradle.testclusters;
 
 import org.elasticsearch.GradleServicesAdapter;
 import org.elasticsearch.gradle.Distribution;
+import org.elasticsearch.gradle.FileSupplier;
 import org.elasticsearch.gradle.Version;
 import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Project;
@@ -143,6 +144,16 @@ public class ElasticsearchCluster implements TestClusterConfiguration {
         nodes.all(each -> each.keystore(key, valueSupplier));
     }
 
+    @Override
+    public void keystore(String key, File value) {
+        nodes.all(each -> each.keystore(key, value));
+    }
+
+    @Override
+    public void keystore(String key, FileSupplier valueSupplier) {
+        nodes.all(each -> each.keystore(key, valueSupplier));
+    }
+
     @Override
     public void setting(String key, String value) {
         nodes.all(each -> each.setting(key, value));

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -20,6 +20,7 @@ package org.elasticsearch.gradle.testclusters;
 
 import org.elasticsearch.GradleServicesAdapter;
 import org.elasticsearch.gradle.Distribution;
+import org.elasticsearch.gradle.FileSupplier;
 import org.elasticsearch.gradle.OS;
 import org.elasticsearch.gradle.Version;
 import org.gradle.api.logging.Logger;
@@ -63,7 +64,8 @@ public class ElasticsearchNode implements TestClusterConfiguration {
     private static final int NODE_UP_TIMEOUT = 60;
     private static final TimeUnit NODE_UP_TIMEOUT_UNIT = TimeUnit.SECONDS;
     private static final List<String> OVERRIDABLE_SETTINGS = Arrays.asList(
-        "path.repo"
+        "path.repo",
+        "discovery.seed_providers"
     );
 
     private final String path;
@@ -79,6 +81,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
     private final List<File> modules = new ArrayList<>();
     private final Map<String, Supplier<CharSequence>> settings = new LinkedHashMap<>();
     private final Map<String, Supplier<CharSequence>> keystoreSettings = new LinkedHashMap<>();
+    private final Map<String, FileSupplier> keystoreFiles = new LinkedHashMap<>();
     private final Map<String, Supplier<CharSequence>> systemProperties = new LinkedHashMap<>();
     private final Map<String, Supplier<CharSequence>> environment = new LinkedHashMap<>();
     private final Map<String, File> extraConfigFiles = new HashMap<>();
@@ -171,6 +174,19 @@ public class ElasticsearchNode implements TestClusterConfiguration {
         addSupplier("Keystore", keystoreSettings, key, valueSupplier);
     }
 
+    @Override
+    public void keystore(String key, File value) {
+        requireNonNull(value, "keystore value was null when configuring test cluster`" + this + "`");
+        keystore(key, () -> value);
+    }
+
+    @Override
+    public void keystore(String key, FileSupplier valueSupplier) {
+        requireNonNull(key, "Keystore" + " key was null when configuring test cluster `" + this + "`");
+        requireNonNull(valueSupplier, "Keystore" + " value supplier was null when configuring test cluster `" + this + "`");
+        keystoreFiles.put(key, valueSupplier);
+    }
+
     @Override
     public void setting(String key, String value) {
         addSupplier("Settings", settings, key, value);
@@ -281,12 +297,22 @@ public class ElasticsearchNode implements TestClusterConfiguration {
             "install", "--batch", plugin.toString())
         );
 
-        if (keystoreSettings.isEmpty() == false) {
-            checkSuppliers("Keystore", keystoreSettings);
+        if (keystoreSettings.isEmpty() == false || keystoreFiles.isEmpty() == false) {
             runElaticsearchBinScript("elasticsearch-keystore", "create");
-            keystoreSettings.forEach((key, value) -> {
-                runElaticsearchBinScriptWithInput(value.get().toString(), "elasticsearch-keystore", "add", "-x", key);
-            });
+
+            checkSuppliers("Keystore", keystoreSettings);
+            keystoreSettings.forEach((key, value) ->
+                runElaticsearchBinScriptWithInput(value.get().toString(), "elasticsearch-keystore", "add", "-x", key)
+            );
+
+            for (Map.Entry<String, FileSupplier> entry : keystoreFiles.entrySet()) {
+                File file = entry.getValue().get();
+                requireNonNull(file, "supplied keystoreFile was null when configuring " + this);
+                if (file.exists() == false) {
+                    throw new TestClustersException("supplied keystore file " + file + " does not exist, require for " + this);
+                }
+                runElaticsearchBinScript("elasticsearch-keystore", "add-file", entry.getKey(), file.getAbsolutePath());
+            }
         }
 
         installModules();

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java

@@ -19,6 +19,7 @@
 package org.elasticsearch.gradle.testclusters;
 
 import org.elasticsearch.gradle.Distribution;
+import org.elasticsearch.gradle.FileSupplier;
 import org.gradle.api.logging.Logging;
 import org.slf4j.Logger;
 
@@ -47,6 +48,10 @@ public interface TestClusterConfiguration {
 
     void keystore(String key, Supplier<CharSequence> valueSupplier);
 
+    void keystore(String key, File value);
+
+    void keystore(String key, FileSupplier valueSupplier);
+
     void setting(String key, String value);
 
     void setting(String key, Supplier<CharSequence> valueSupplier);

plugins/build.gradle

@@ -17,10 +17,13 @@
  * under the License.
  */
 
+subprojects  {
+  apply plugin: 'elasticsearch.testclusters'       
+}
+
 // only configure immediate children of plugins dir
 configure(subprojects.findAll { it.parent.path == project.path }) {
   group = 'org.elasticsearch.plugin'
-  apply plugin: 'elasticsearch.testclusters'       
   apply plugin: 'elasticsearch.esplugin'
 
   esplugin {

plugins/discovery-ec2/qa/amazon-ec2/build.gradle

@@ -29,14 +29,13 @@ dependencies {
 }
 
 final int ec2NumberOfNodes = 3
-File ec2DiscoveryFile =  new File(project.buildDir, 'generated-resources/nodes.uri')
 
 /** A task to start the AmazonEC2Fixture which emulates an EC2 service **/
 task ec2Fixture(type: AntFixture) {
     dependsOn compileTestJava
     env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }"
     executable = new File(project.runtimeJavaHome, 'bin/java')
-    args 'org.elasticsearch.discovery.ec2.AmazonEC2Fixture', baseDir, ec2DiscoveryFile.absolutePath
+    args 'org.elasticsearch.discovery.ec2.AmazonEC2Fixture', baseDir, "${buildDir}/testclusters/integTest-1/config/unicast_hosts.txt"
 }
 
 Map<String, Object> expansions = [
@@ -48,28 +47,20 @@ processTestResources {
     MavenFilteringHack.filter(it, expansions)
 }
 
-integTestCluster {
-    dependsOn ec2Fixture
-    numNodes = ec2NumberOfNodes
-    plugin ':plugins:discovery-ec2'
-    keystoreSetting 'discovery.ec2.access_key', 'ec2_integration_test_access_key'
-    keystoreSetting 'discovery.ec2.secret_key', 'ec2_integration_test_secret_key'
+integTest {
+    dependsOn ec2Fixture, project(':plugins:discovery-ec2').bundlePlugin
+}
+
+testClusters.integTest {
+    numberOfNodes = ec2NumberOfNodes
+    plugin file(project(':plugins:discovery-ec2').bundlePlugin.archiveFile)
+
+    keystore 'discovery.ec2.access_key', 'ec2_integration_test_access_key'
+    keystore 'discovery.ec2.secret_key', 'ec2_integration_test_secret_key'
+    
     setting 'discovery.seed_providers', 'ec2'
     setting 'network.host', '_ec2_'
-    setting 'discovery.ec2.endpoint', "http://${-> ec2Fixture.addressAndPort}"
-    systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", "http://${-> ec2Fixture.addressAndPort}"
+    setting 'discovery.ec2.endpoint', { "http://${ec2Fixture.addressAndPort}" }
     
-    unicastTransportUri = { seedNode, node, ant -> return null }
-
-    waitCondition = { node, ant ->
-        ec2DiscoveryFile.parentFile.mkdirs()
-        ec2DiscoveryFile.setText(integTest.nodes.collect { n -> "${n.transportUri()}" }.join('\n'), 'UTF-8')
-
-        File tmpFile = new File(node.cwd, 'wait.success')
-        ant.get(src: "http://${node.httpUri()}/",
-                dest: tmpFile.toString(),
-                ignoreerrors: true,
-                retries: 10)
-        return tmpFile.exists()
-    }
+    systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", { "http://${ec2Fixture.addressAndPort}" }
 }

plugins/discovery-gce/qa/gce/build.gradle

@@ -24,7 +24,6 @@ apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 final int gceNumberOfNodes = 3
-File gceDiscoveryFile =  new File(project.buildDir, 'generated-resources/nodes.uri')
 
 dependencies {
     testCompile project(path: ':plugins:discovery-gce', configuration: 'runtime')
@@ -35,7 +34,7 @@ task gceFixture(type: AntFixture) {
     dependsOn compileTestJava
     env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }"
     executable = new File(project.runtimeJavaHome, 'bin/java')
-    args 'org.elasticsearch.cloud.gce.GCEFixture', baseDir, gceDiscoveryFile.getAbsolutePath()
+    args 'org.elasticsearch.cloud.gce.GCEFixture', baseDir, "${buildDir}/testclusters/integTest-1/config/unicast_hosts.txt"
 }
 
 Map<String, Object> expansions = [
@@ -47,34 +46,21 @@ processTestResources {
     MavenFilteringHack.filter(it, expansions)
 }
 
-integTestCluster {
-    dependsOn gceFixture
-    numNodes = gceNumberOfNodes
-    plugin ':plugins:discovery-gce'
-    setting 'discovery.seed_providers', 'gce'
+integTest {
+    dependsOn gceFixture, project(':plugins:discovery-gce').bundlePlugin
+}
 
+testClusters.integTest {
+    numberOfNodes = gceNumberOfNodes
+    plugin file(project(':plugins:discovery-gce').bundlePlugin.archiveFile)
     // use gce fixture for Auth calls instead of http://metadata.google.internal
-    integTestCluster.environment 'GCE_METADATA_HOST', "http://${-> gceFixture.addressAndPort}"
-
+    environment 'GCE_METADATA_HOST', { "http://${gceFixture.addressAndPort}" }
     // allows to configure hidden settings (`cloud.gce.host` and `cloud.gce.root_url`)
     systemProperty 'es.allow_reroute_gce_settings', 'true'
 
+    setting 'discovery.seed_providers', 'gce'
     // use gce fixture for metadata server calls instead of http://metadata.google.internal
-    setting 'cloud.gce.host', "http://${-> gceFixture.addressAndPort}"
+    setting 'cloud.gce.host', { "http://${gceFixture.addressAndPort}" }
     // use gce fixture for API calls instead of https://www.googleapis.com
-    setting 'cloud.gce.root_url', "http://${-> gceFixture.addressAndPort}"
-
-    unicastTransportUri = { seedNode, node, ant -> return null }
-
-    waitCondition = { node, ant ->
-        gceDiscoveryFile.parentFile.mkdirs()
-        gceDiscoveryFile.setText(integTest.nodes.collect { n -> "${n.transportUri()}" }.join('\n'), 'UTF-8')
-
-        File tmpFile = new File(node.cwd, 'wait.success')
-        ant.get(src: "http://${node.httpUri()}/",
-                dest: tmpFile.toString(),
-                ignoreerrors: true,
-                retries: 10)
-        return tmpFile.exists()
-    }
+    setting 'cloud.gce.root_url', { "http://${gceFixture.addressAndPort}" }
 }

plugins/repository-azure/qa/microsoft-azure-storage/build.gradle

@@ -23,10 +23,6 @@ import org.elasticsearch.gradle.test.AntFixture
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
-integTestCluster {
-    plugin ':plugins:repository-azure'
-}
-
 boolean useFixture = false
 
 String azureAccount = System.getenv("azure_storage_account")
@@ -60,16 +56,21 @@ processTestResources {
     MavenFilteringHack.filter(it, expansions)
 }
 
-integTestCluster {
-    keystoreSetting 'azure.client.integration_test.account', azureAccount
-    keystoreSetting 'azure.client.integration_test.key', azureKey
+integTest {
+    dependsOn project(':plugins:repository-azure').bundlePlugin
+} 
+
+testClusters.integTest {
+    plugin file(project(':plugins:repository-azure').bundlePlugin.archiveFile)
+    keystore 'azure.client.integration_test.account', azureAccount
+    keystore 'azure.client.integration_test.key', azureKey
 
     if (useFixture) {
-        dependsOn azureStorageFixture
+        tasks.integTest.dependsOn azureStorageFixture
         // Use a closure on the string to delay evaluation until tests are executed. The endpoint_suffix is used
         // in a hacky way to change the protocol and endpoint. We must fix that.
         setting 'azure.client.integration_test.endpoint_suffix',
-                "ignored;DefaultEndpointsProtocol=http;BlobEndpoint=http://${ -> azureStorageFixture.addressAndPort }"
+                { "ignored;DefaultEndpointsProtocol=http;BlobEndpoint=http://${azureStorageFixture.addressAndPort }" }
     } else {
         println "Using an external service to test the repository-azure plugin"
     }

plugins/repository-gcs/qa/google-cloud-storage/build.gradle

@@ -26,10 +26,6 @@ import java.security.KeyPairGenerator
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
-integTestCluster {
-    plugin ':plugins:repository-gcs'
-}
-
 boolean useFixture = false
 
 String gcsServiceAccount = System.getenv("google_storage_service_account")
@@ -87,14 +83,20 @@ processTestResources {
     MavenFilteringHack.filter(it, expansions)
 }
 
-integTestCluster {
-    keystoreFile 'gcs.client.integration_test.credentials_file', "${serviceAccountFile.absolutePath}"
+integTest {
+    dependsOn project(':plugins:repository-gcs').bundlePlugin
+}
+
+testClusters.integTest {
+    plugin file(project(':plugins:repository-gcs').bundlePlugin.archiveFile)
+
+    keystore 'gcs.client.integration_test.credentials_file', serviceAccountFile
 
     if (useFixture) {
-        dependsOn createServiceAccountFile, googleCloudStorageFixture
+        tasks.integTest.dependsOn createServiceAccountFile, googleCloudStorageFixture
         /* Use a closure on the string to delay evaluation until tests are executed */
-        setting 'gcs.client.integration_test.endpoint', "http://${ -> googleCloudStorageFixture.addressAndPort }"
-        setting 'gcs.client.integration_test.token_uri', "http://${ -> googleCloudStorageFixture.addressAndPort }/o/oauth2/token"
+        setting 'gcs.client.integration_test.endpoint', { "http://${googleCloudStorageFixture.addressAndPort}" }
+        setting 'gcs.client.integration_test.token_uri', { "http://${googleCloudStorageFixture.addressAndPort}/o/oauth2/token" }
     } else {
         println "Using an external service to test the repository-gcs plugin"
     }

x-pack/qa/security-migrate-tests/build.gradle

@@ -30,6 +30,7 @@ integTestCluster {
             retries: 10)
     return tmpFile.exists()
   }
+  // TODO: systemProperty('tests.cluster', "${-> cluster.transportPortURI }") when migerating to testclusters
 }
 
 testingConventions {