Testclusters: support for security and convert example plugins (#41864)

testclusters detect from settings that security is enabled
if a user is not specified using the DSL introduced in this PR, a default one is created
the appropriate wait conditions are used authenticating with the first user defined in the DSL ( or the default user ).
an example DSL to create a user is user username:"test_user" password:"x-pack-test-password" role: "superuser" all keys are optional and default to the values shown in this example

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy

@@ -70,7 +70,7 @@ class RestIntegTestTask extends DefaultTask {
             project.testClusters {
                 "$name" {
                     distribution = 'INTEG_TEST'
-                    version = project.version
+                    version = VersionProperties.elasticsearch
                     javaHome = project.file(project.ext.runtimeJavaHome)
                 }
             }

buildSrc/src/main/java/org/elasticsearch/gradle/http/WaitForHttpResource.java

@@ -123,8 +123,7 @@ public class WaitForHttpResource {
             if (System.nanoTime() < waitUntil) {
                 Thread.sleep(sleep);
             } else {
-                logger.error("Failed to access url [{}]", url, failure);
-                return false;
+                throw failure;
             }
         }
     }

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java

@@ -22,23 +22,22 @@ import org.elasticsearch.GradleServicesAdapter;
 import org.elasticsearch.gradle.Distribution;
 import org.elasticsearch.gradle.FileSupplier;
 import org.elasticsearch.gradle.Version;
+import org.elasticsearch.gradle.http.WaitForHttpResource;
 import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Project;
 import org.gradle.api.logging.Logger;
 import org.gradle.api.logging.Logging;
 
-import java.io.BufferedReader;
 import java.io.File;
 import java.io.IOException;
-import java.io.InputStreamReader;
 import java.io.UncheckedIOException;
-import java.net.HttpURLConnection;
 import java.net.URI;
-import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
+import java.security.GeneralSecurityException;
 import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -75,6 +74,8 @@ public class ElasticsearchCluster implements TestClusterConfiguration {
                 services, artifactsExtractDir, workingDirBase
             )
         );
+
+        addWaitForClusterHealth();
     }
 
     public void setNumberOfNodes(int numberOfNodes) {
@@ -219,6 +220,11 @@ public class ElasticsearchCluster implements TestClusterConfiguration {
         nodes.all(node -> node.extraConfigFile(destination, from));
     }
 
+    @Override
+    public void user(Map<String, String> userSpec) {
+        nodes.all(node -> node.user(userSpec));
+    }
+
     private void writeUnicastHostsFiles() {
         String unicastUris = nodes.stream().flatMap(node -> node.getAllTransportPortURI().stream()).collect(Collectors.joining("\n"));
         nodes.forEach(node -> {
@@ -262,9 +268,6 @@ public class ElasticsearchCluster implements TestClusterConfiguration {
         writeUnicastHostsFiles();
 
         LOGGER.info("Starting to wait for cluster to form");
-        addWaitForUri(
-            "cluster health yellow", "/_cluster/health?wait_for_nodes=>=" + nodes.size()  + "&wait_for_status=yellow"
-        );
         waitForConditions(waitConditions, startedAt, CLUSTER_UP_TIMEOUT, CLUSTER_UP_TIMEOUT_UNIT, this);
     }
 
@@ -293,21 +296,25 @@ public class ElasticsearchCluster implements TestClusterConfiguration {
         return getFirstNode();
     }
 
-    private void addWaitForUri(String description, String uri) {
-        waitConditions.put(description, (node) -> {
+    private void addWaitForClusterHealth() {
+        waitConditions.put("cluster health yellow", (node) -> {
             try {
-                URL url = new URL("http://" + getFirstNode().getHttpSocketURI() + uri);
-                HttpURLConnection con = (HttpURLConnection) url.openConnection();
-                con.setRequestMethod("GET");
-                con.setConnectTimeout(500);
-                con.setReadTimeout(500);
-                try (BufferedReader reader = new BufferedReader(new InputStreamReader(con.getInputStream()))) {
-                    String response = reader.lines().collect(Collectors.joining("\n"));
-                    LOGGER.info("{} -> {} ->\n{}", this, uri, response);
+                WaitForHttpResource wait = new WaitForHttpResource(
+                    "http", getFirstNode().getHttpSocketURI(), nodes.size()
+                );
+                List<Map<String, String>> credentials = getFirstNode().getCredentials();
+                if (getFirstNode().getCredentials().isEmpty() == false) {
+                    wait.setUsername(credentials.get(0).get("useradd"));
+                    wait.setPassword(credentials.get(0).get("-p"));
                 }
-                return true;
+                return wait.wait(500);
             } catch (IOException e) {
                 throw new IllegalStateException("Connection attempt to " + this + " failed", e);
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                throw new TestClustersException("Interrupted while waiting for " + this, e);
+            } catch (GeneralSecurityException e) {
+                throw new RuntimeException("security exception", e);
             }
         });
     }

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -38,6 +38,7 @@ import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
@@ -86,6 +87,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
     private final Map<String, Supplier<CharSequence>> environment = new LinkedHashMap<>();
     private final Map<String, File> extraConfigFiles = new HashMap<>();
     final LinkedHashMap<String, String> defaultConfig = new LinkedHashMap<>();
+    private final List<Map<String, String>> credentials = new ArrayList<>();
 
     private final Path confPathRepo;
     private final Path configFile;
@@ -117,8 +119,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
         esStdoutFile = confPathLogs.resolve("es.stdout.log");
         esStderrFile = confPathLogs.resolve("es.stderr.log");
         tmpDir = workingDir.resolve("tmp");
-        waitConditions.put("http ports file", node -> Files.exists(((ElasticsearchNode) node).httpPortsFile));
-        waitConditions.put("transport ports file", node -> Files.exists(((ElasticsearchNode)node).transportPortFile));
+        waitConditions.put("ports files", this::checkPortsFilesExistWithDelay);
     }
 
     public String getName() {
@@ -319,9 +320,25 @@ public class ElasticsearchNode implements TestClusterConfiguration {
 
         copyExtraConfigFiles();
 
+        if (isSettingMissingOrTrue("xpack.security.enabled")) {
+            if (credentials.isEmpty()) {
+                user(Collections.emptyMap());
+            }
+            credentials.forEach(paramMap -> runElaticsearchBinScript(
+                "elasticsearch-users",
+                    paramMap.entrySet().stream()
+                        .flatMap(entry -> Stream.of(entry.getKey(), entry.getValue()))
+                        .toArray(String[]::new)
+            ));
+        }
+
         startElasticsearchProcess();
     }
 
+    private boolean isSettingMissingOrTrue(String name) {
+        return Boolean.valueOf(settings.getOrDefault(name, () -> "false").get().toString());
+    }
+
     private void copyExtraConfigFiles() {
         extraConfigFiles.forEach((destination, from) -> {
                 if (Files.exists(from.toPath()) == false) {
@@ -375,6 +392,22 @@ public class ElasticsearchNode implements TestClusterConfiguration {
         extraConfigFiles.put(destination, from);
     }
 
+    @Override
+    public void user(Map<String, String> userSpec) {
+        Set<String> keys = new HashSet<>(userSpec.keySet());
+        keys.remove("username");
+        keys.remove("password");
+        keys.remove("role");
+        if (keys.isEmpty() == false) {
+            throw new TestClustersException("Unknown keys in user definition " + keys + " for " + this);
+        }
+        Map<String,String> cred = new LinkedHashMap<>();
+        cred.put("useradd", userSpec.getOrDefault("username","test_user"));
+        cred.put("-p", userSpec.getOrDefault("password","x-pack-test-password"));
+        cred.put("-r", userSpec.getOrDefault("role", "superuser"));
+        credentials.add(cred);
+    }
+
     private void runElaticsearchBinScriptWithInput(String input, String tool, String... args) {
         try (InputStream byteArrayInputStream = new ByteArrayInputStream(input.getBytes(StandardCharsets.UTF_8))) {
             services.loggedExec(spec -> {
@@ -752,4 +785,21 @@ public class ElasticsearchNode implements TestClusterConfiguration {
     public String toString() {
         return "node{" + path + ":" + name + "}";
     }
+
+    List<Map<String, String>> getCredentials() {
+        return credentials;
+    }
+
+    private boolean checkPortsFilesExistWithDelay(TestClusterConfiguration node) {
+        if (Files.exists(httpPortsFile) && Files.exists(transportPortFile)) {
+            return true;
+        }
+        try {
+            Thread.sleep(500);
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            throw new TestClustersException("Interrupted while waiting for ports files", e);
+        }
+        return Files.exists(httpPortsFile) && Files.exists(transportPortFile);
+    }
 }

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java

@@ -27,6 +27,7 @@ import java.io.File;
 import java.net.URI;
 import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Predicate;
 import java.util.function.Supplier;
@@ -72,6 +73,8 @@ public interface TestClusterConfiguration {
 
     void extraConfigFile(String destination, File from);
 
+    void user(Map<String, String> userSpec);
+
     String getHttpSocketURI();
 
     String getTransportPortURI();
@@ -108,7 +111,7 @@ public interface TestClusterConfiguration {
                         break;
                     }
                 } catch (TestClustersException e) {
-                    throw new TestClustersException(e);
+                    throw e;
                 } catch (Exception e) {
                     if (lastException == null) {
                         lastException = e;
@@ -116,12 +119,6 @@ public interface TestClusterConfiguration {
                         lastException = e;
                     }
                 }
-                try {
-                    Thread.sleep(500);
-                }
-                catch (InterruptedException e) {
-                    Thread.currentThread().interrupt();
-                }
             }
             if (conditionMet == false) {
                 String message = "`" + context + "` failed to wait for " + description + " after " +

distribution/archives/build.gradle

@@ -295,6 +295,10 @@ subprojects {
   }
 }
 
+subprojects {
+  group = "org.elasticsearch.distribution.${name.startsWith("oss-") ? "oss" : "default"}"
+}
+
 /*****************************************************************************
  *                            Rest test config                               *
  *****************************************************************************/
@@ -302,6 +306,8 @@ configure(subprojects.findAll { it.name == 'integ-test-zip' }) {
   apply plugin: 'elasticsearch.standalone-rest-test'
   apply plugin: 'elasticsearch.rest-test'
 
+  group = "org.elasticsearch.distribution.integ-test-zip"
+
   integTest {
     includePackaged = true
   }
@@ -321,23 +327,14 @@ configure(subprojects.findAll { it.name == 'integ-test-zip' }) {
     inputs.properties(project(':distribution').restTestExpansions)
     MavenFilteringHack.filter(it, project(':distribution').restTestExpansions)
   }
-}
 
-/*****************************************************************************
- *                              Maven config                                 *
- *****************************************************************************/
-configure(subprojects.findAll { it.name.contains('zip') }) {
-  // only zip distributions go to maven
+
+  // The integ-test-distribution is published to maven
   BuildPlugin.configurePomGeneration(project)
   apply plugin: 'nebula.info-scm'
   apply plugin: 'nebula.maven-base-publish'
   apply plugin: 'nebula.maven-scm'
 
-  // note: the group must be correct before applying the nexus plugin, or
-  // it will capture the wrong value...
-  String subgroup = project.name == 'integ-test-zip' ? 'integ-test-zip' : 'zip'
-  project.group = "org.elasticsearch.distribution.${subgroup}"
-
   // make the pom file name use elasticsearch instead of the project name
   archivesBaseName = "elasticsearch${it.name.contains('oss') ? '-oss' : ''}"
 
@@ -378,3 +375,4 @@ configure(subprojects.findAll { it.name.contains('zip') }) {
     }
   }
 }
+

plugins/examples/custom-settings/build.gradle

@@ -16,6 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.esplugin'
 
 esplugin {
@@ -26,7 +27,7 @@ esplugin {
   noticeFile rootProject.file('NOTICE.txt')
 }
 
-integTestCluster {
+testClusters.integTest {
   // Adds a setting in the Elasticsearch keystore before running the integration tests
-  keystoreSetting 'custom.secured', 'password'
-}
+  keystore 'custom.secured', 'password'
+}

plugins/examples/custom-suggester/build.gradle

@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.esplugin'
 
 esplugin {
@@ -27,8 +27,8 @@ esplugin {
     noticeFile rootProject.file('NOTICE.txt')
 }
 
-integTestCluster {
-    numNodes = 2
+testClusters.integTest {
+    numberOfNodes = 2
 }
 
 // this plugin has no unit tests, only rest tests

plugins/examples/painless-whitelist/build.gradle

@@ -16,6 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.esplugin'
 
 esplugin {
@@ -31,8 +32,8 @@ dependencies {
   compileOnly "org.elasticsearch.plugin:elasticsearch-scripting-painless-spi:${versions.elasticsearch}"
 }
 
-if (System.getProperty('tests.distribution') == null) {
-  integTestCluster.distribution = 'oss'
+testClusters.integTest {
+  distribution = 'oss'
 }
 
 test.enabled = false

plugins/examples/rescore/build.gradle

@@ -16,6 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.esplugin'
 
 esplugin {

plugins/examples/rest-handler/build.gradle

@@ -16,6 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.esplugin'
 
 esplugin {
@@ -36,11 +37,11 @@ task exampleFixture(type: org.elasticsearch.gradle.test.AntFixture) {
   args 'org.elasticsearch.example.resthandler.ExampleFixture', baseDir, 'TEST'
 }
 
-integTestCluster {
+integTest {
   dependsOn exampleFixture
-}
-integTestRunner {
-  nonInputProperties.systemProperty 'external.address', "${ -> exampleFixture.addressAndPort }"
+  runner {
+    nonInputProperties.systemProperty 'external.address', "${ -> exampleFixture.addressAndPort }"  
+  }
 }
 
 testingConventions.naming {

plugins/examples/script-expert-scoring/build.gradle

@@ -16,6 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.esplugin'
 
 esplugin {

plugins/examples/security-authorization-engine/build.gradle

@@ -1,3 +1,4 @@
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.esplugin'
 
 esplugin {
@@ -14,15 +15,14 @@ dependencies {
   testCompile "org.elasticsearch.client:x-pack-transport:${versions.elasticsearch}"
 }
 
-
-integTestRunner {
+integTest {
+  dependsOn buildZip
+  runner {
     systemProperty 'tests.security.manager', 'false'
+  }
 }
 
-integTestCluster {
-  dependsOn buildZip
-  distribution = 'default'
-
+testClusters.integTest {
   setting 'xpack.security.enabled', 'true'
   setting 'xpack.ilm.enabled', 'false'
   setting 'xpack.ml.enabled', 'false'
@@ -34,17 +34,7 @@ integTestCluster {
   // processors are being used that are in ingest-common module.
   distribution = 'default'
 
-  setupCommand 'setupDummyUser',
-               'bin/elasticsearch-users', 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'custom_superuser'
-  waitCondition = { node, ant ->
-    File tmpFile = new File(node.cwd, 'wait.success')
-    ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
-            dest: tmpFile.toString(),
-            username: 'test_user',
-            password: 'x-pack-test-password',
-            ignoreerrors: true,
-            retries: 10)
-    return tmpFile.exists()
-  }
+  user role: 'custom_superuser'
 }
+
 check.dependsOn integTest

x-pack/qa/security-setup-password-tests/build.gradle

@@ -10,6 +10,7 @@ dependencies {
 
 integTestRunner {
     systemProperty 'tests.security.manager', 'false'
+    // TODO add  tests.config.dir = {cluster.singleNode().getConfigDir()} when converting to testclusters
 }
 
 integTestCluster {