TestClusters: Convert docs (#42100)

* TestClusters: Convert docs

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy

@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.gradle.doc
 
+import org.elasticsearch.gradle.OS
 import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.test.ClusterFormationTasks
@@ -32,10 +33,13 @@ public class DocsTestPlugin extends RestTestPlugin {
 
     @Override
     public void apply(Project project) {
+        project.pluginManager.apply('elasticsearch.testclusters')
         project.pluginManager.apply('elasticsearch.standalone-rest-test')
         super.apply(project)
+        String distribution = System.getProperty('tests.distribution', 'default')
         // The distribution can be configured with -Dtests.distribution on the command line
-        project.integTestCluster.distribution = System.getProperty('tests.distribution', 'default')
+        project.testClusters.integTest.distribution = distribution.toUpperCase()
+        project.testClusters.integTest.nameCustomization = { it.replace("integTest", "node") }
         // Docs are published separately so no need to assemble
         project.tasks.assemble.enabled = false
         Map<String, String> defaultSubstitutions = [
@@ -46,8 +50,8 @@ public class DocsTestPlugin extends RestTestPlugin {
             '\\{version\\}': Version.fromString(VersionProperties.elasticsearch).toString(),
             '\\{version_qualified\\}': VersionProperties.elasticsearch,
             '\\{lucene_version\\}' : VersionProperties.lucene.replaceAll('-snapshot-\\w+$', ''),
-            '\\{build_flavor\\}' : project.integTestCluster.distribution,
-            '\\{build_type\\}' : ClusterFormationTasks.getOs().equals("windows") ? "zip" : "tar",
+            '\\{build_flavor\\}' : distribution,
+            '\\{build_type\\}' : OS.conditionalString().onWindows({"zip"}).onUnix({"tar"}).supply(),
         ]
         Task listSnippets = project.tasks.create('listSnippets', SnippetsTask)
         listSnippets.group 'Docs'

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java

@@ -42,6 +42,7 @@ import java.util.Objects;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.BiConsumer;
+import java.util.function.Function;
 import java.util.function.Predicate;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
@@ -70,7 +71,7 @@ public class ElasticsearchCluster implements TestClusterConfiguration {
         this.nodes = project.container(ElasticsearchNode.class);
         this.nodes.add(
             new ElasticsearchNode(
-                path, clusterName + "-1",
+                path, clusterName + "-0",
                 services, artifactsExtractDir, workingDirBase
             )
         );
@@ -91,7 +92,7 @@ public class ElasticsearchCluster implements TestClusterConfiguration {
             );
         }
 
-        for (int i = nodes.size() + 1 ; i <= numberOfNodes; i++) {
+        for (int i = nodes.size() ; i < numberOfNodes; i++) {
             this.nodes.add(new ElasticsearchNode(
                 path, clusterName + "-" + i, services, artifactsExtractDir, workingDirBase
             ));
@@ -99,7 +100,7 @@ public class ElasticsearchCluster implements TestClusterConfiguration {
     }
 
     private ElasticsearchNode getFirstNode() {
-        return nodes.getAt(clusterName + "-1");
+        return nodes.getAt(clusterName + "-0");
     }
 
     public int getNumberOfNodes() {
@@ -276,6 +277,11 @@ public class ElasticsearchCluster implements TestClusterConfiguration {
         nodes.forEach(each -> each.stop(tailLogs));
     }
 
+    @Override
+    public void setNameCustomization(Function<String, String> nameCustomization) {
+        nodes.all(each -> each.setNameCustomization(nameCustomization));
+    }
+
     @Override
     public boolean isProcessAlive() {
         return nodes.stream().noneMatch(node -> node.isProcessAlive() == false);

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -50,6 +50,7 @@ import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Function;
 import java.util.function.Predicate;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
@@ -103,6 +104,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
     private String version;
     private File javaHome;
     private volatile Process esProcess;
+    private Function<String, String> nameCustomization = Function.identity();
 
     ElasticsearchNode(String path, String name, GradleServicesAdapter services, File artifactsExtractDir, File workingDirBase) {
         this.path = path;
@@ -123,7 +125,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
     }
 
     public String getName() {
-        return name;
+        return nameCustomization.apply(name);
     }
 
     public String getVersion() {
@@ -536,6 +538,11 @@ public class ElasticsearchNode implements TestClusterConfiguration {
         esProcess = null;
     }
 
+    @Override
+    public void setNameCustomization(Function<String, String> nameCustomizer) {
+        this.nameCustomization = nameCustomizer;
+    }
+
     private void stopHandle(ProcessHandle processHandle, boolean forcibly) {
         // Stop all children first, ES could actually be a child when there's some wrapper process like on Windows.
         if (processHandle.isAlive() == false) {
@@ -656,7 +663,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
     }
 
     private void createConfiguration()  {
-        defaultConfig.put("node.name", safeName(name));
+        defaultConfig.put("node.name", nameCustomization.apply(safeName(name)));
         defaultConfig.put("path.repo", confPathRepo.toAbsolutePath().toString());
         defaultConfig.put("path.data", confPathData.toAbsolutePath().toString());
         defaultConfig.put("path.logs", confPathLogs.toAbsolutePath().toString());

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java

@@ -29,6 +29,7 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
 import java.util.function.Predicate;
 import java.util.function.Supplier;
 
@@ -85,6 +86,8 @@ public interface TestClusterConfiguration {
 
     void stop(boolean tailLogs);
 
+    void setNameCustomization(Function<String, String> nameSupplier);
+
     default void waitForConditions(
         LinkedHashMap<String, Predicate<TestClusterConfiguration>> waitConditions,
         long startedAtMillis,

buildSrc/src/test/java/org/elasticsearch/gradle/testclusters/TestClustersPluginIT.java

@@ -86,8 +86,8 @@ public class TestClustersPluginIT extends GradleIntegrationTestCase {
         assertOutputContains(
             result.getOutput(),
             "> Task :user1",
-            "Starting `node{::myTestCluster-1}`",
-            "Stopping `node{::myTestCluster-1}`"
+            "Starting `node{::myTestCluster-0}`",
+            "Stopping `node{::myTestCluster-0}`"
         );
     }
 
@@ -104,22 +104,22 @@ public class TestClustersPluginIT extends GradleIntegrationTestCase {
         assertStartedAndStoppedOnce(result);
         assertOutputOnlyOnce(
             result.getOutput(),
-            "Starting `node{:alpha:myTestCluster-1}`",
-            "Stopping `node{::myTestCluster-1}`"
+            "Starting `node{:alpha:myTestCluster-0}`",
+            "Stopping `node{::myTestCluster-0}`"
         );
         assertOutputOnlyOnce(
             result.getOutput(),
-            "Starting `node{::myTestCluster-1}`",
-            "Stopping `node{:bravo:myTestCluster-1}`"
+            "Starting `node{::myTestCluster-0}`",
+            "Stopping `node{:bravo:myTestCluster-0}`"
         );
     }
 
     public void testReleased() {
         BuildResult result = getTestClustersRunner("testReleased").build();
         assertTaskSuccessful(result, ":testReleased");
-        assertStartedAndStoppedOnce(result, "releasedVersionDefault-1");
-        assertStartedAndStoppedOnce(result, "releasedVersionOSS-1");
-        assertStartedAndStoppedOnce(result, "releasedVersionIntegTest-1");
+        assertStartedAndStoppedOnce(result, "releasedVersionDefault-0");
+        assertStartedAndStoppedOnce(result, "releasedVersionOSS-0");
+        assertStartedAndStoppedOnce(result, "releasedVersionIntegTest-0");
     }
 
     public void testIncremental() {
@@ -143,7 +143,7 @@ public class TestClustersPluginIT extends GradleIntegrationTestCase {
         assertStartedAndStoppedOnce(result);
         assertOutputContains(
             result.getOutput(),
-            "Stopping `node{::myTestCluster-1}`, tailLogs: true",
+            "Stopping `node{::myTestCluster-0}`, tailLogs: true",
             "Execution failed for task ':itAlwaysFails'."
         );
     }
@@ -155,7 +155,7 @@ public class TestClustersPluginIT extends GradleIntegrationTestCase {
         assertStartedAndStoppedOnce(result);
         assertOutputContains(
             result.getOutput(),
-            "Stopping `node{::myTestCluster-1}`, tailLogs: true",
+            "Stopping `node{::myTestCluster-0}`, tailLogs: true",
             "Execution failed for task ':itAlwaysFails'."
         );
     }
@@ -165,7 +165,7 @@ public class TestClustersPluginIT extends GradleIntegrationTestCase {
         assertTaskFailed(result, ":illegalConfigAlter");
         assertOutputContains(
             result.getOutput(),
-            "Configuration for node{::myTestCluster-1} can not be altered, already locked"
+            "Configuration for node{::myTestCluster-0} can not be altered, already locked"
         );
     }
 
@@ -173,9 +173,9 @@ public class TestClustersPluginIT extends GradleIntegrationTestCase {
     public void testMultiNode() {
         BuildResult result = getTestClustersRunner(":multiNode").build();
         assertTaskSuccessful(result, ":multiNode");
+        assertStartedAndStoppedOnce(result, "multiNode-0");
         assertStartedAndStoppedOnce(result, "multiNode-1");
         assertStartedAndStoppedOnce(result, "multiNode-2");
-        assertStartedAndStoppedOnce(result, "multiNode-3");
     }
 
     public void testPluginInstalled() {
@@ -211,7 +211,7 @@ public class TestClustersPluginIT extends GradleIntegrationTestCase {
     }
 
     private void assertStartedAndStoppedOnce(BuildResult result) {
-        assertStartedAndStoppedOnce(result, "myTestCluster-1");
+        assertStartedAndStoppedOnce(result, "myTestCluster-0");
     }
 
 

docs/build.gradle

@@ -1,3 +1,5 @@
+import static org.elasticsearch.gradle.Distribution.DEFAULT
+
 /*
  * Licensed to Elasticsearch under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -35,15 +37,15 @@ buildRestTests.expectedUnconvertedCandidates = [
         'reference/ml/apis/update-snapshot.asciidoc',
 ]
 
-integTestCluster {
-  if ("default".equals(integTestCluster.distribution)) {
+testClusters.integTest {
+  if (singleNode().distribution == DEFAULT) {
     setting 'xpack.license.self_generated.type', 'trial'
   }
 
   // enable regexes in painless so our tests don't complain about example snippets that use them
   setting 'script.painless.regex.enabled', 'true'
   Closure configFile = {
-    extraConfigFile it, "src/test/cluster/config/$it"
+    extraConfigFile it, file("src/test/cluster/config/$it")
   }
   configFile 'analysis/example_word_list.txt'
   configFile 'analysis/hyphenation_patterns.xml'
@@ -52,8 +54,8 @@ integTestCluster {
   configFile 'userdict_ja.txt'
   configFile 'userdict_ko.txt'
   configFile 'KeywordTokenizer.rbbi'
-  extraConfigFile 'hunspell/en_US/en_US.aff', '../server/src/test/resources/indices/analyze/conf_dir/hunspell/en_US/en_US.aff'
-  extraConfigFile 'hunspell/en_US/en_US.dic', '../server/src/test/resources/indices/analyze/conf_dir/hunspell/en_US/en_US.dic'
+  extraConfigFile 'hunspell/en_US/en_US.aff', project(":server").file('src/test/resources/indices/analyze/conf_dir/hunspell/en_US/en_US.aff')
+  extraConfigFile 'hunspell/en_US/en_US.dic', project(":server").file('src/test/resources/indices/analyze/conf_dir/hunspell/en_US/en_US.dic')
   // Whitelist reindexing from the local node so we can test it.
   setting 'reindex.remote.whitelist', '127.0.0.1:*'
 }
@@ -65,10 +67,12 @@ project.rootProject.subprojects.findAll { it.parent.path == ':plugins' }.each {
   if (subproj.path.startsWith(':plugins:repository-')) {
     return
   }
+  // FIXME
   subproj.afterEvaluate { // need to wait until the project has been configured
-    integTestCluster {
-      plugin subproj.path
+    testClusters.integTest {
+      plugin file(subproj.bundlePlugin.archiveFile)
     }
+    tasks.integTest.dependsOn subproj.bundlePlugin
   }
 }
 

docs/reference/cluster/health.asciidoc

@@ -34,7 +34,7 @@ Returns this:
   "active_shards_percent_as_number": 50.0
 }
 --------------------------------------------------
-// TESTRESPONSE[s/testcluster/docs_integTestCluster/]
+// TESTRESPONSE[s/testcluster/integTest/]
 // TESTRESPONSE[s/"number_of_pending_tasks" : 0,/"number_of_pending_tasks" : $body.number_of_pending_tasks,/]
 // TESTRESPONSE[s/"task_max_waiting_in_queue_millis": 0/"task_max_waiting_in_queue_millis": $body.task_max_waiting_in_queue_millis/]
 

docs/reference/getting-started.asciidoc

@@ -301,7 +301,7 @@ And the response:
 epoch      timestamp cluster       status node.total node.data shards pri relo init unassign pending_tasks max_task_wait_time active_shards_percent
 1475247709 17:01:49  elasticsearch green           1         1      0   0    0    0        0             0                  -                100.0%
 --------------------------------------------------
-// TESTRESPONSE[s/1475247709 17:01:49  elasticsearch/\\d+ \\d+:\\d+:\\d+ docs_integTestCluster/]
+// TESTRESPONSE[s/1475247709 17:01:49  elasticsearch/\\d+ \\d+:\\d+:\\d+ integTest/]
 // TESTRESPONSE[s/0             0                  -/0             \\d+                  -/]
 // TESTRESPONSE[_cat]
 

x-pack/docs/build.gradle

@@ -27,54 +27,14 @@ dependencies {
     testCompile project(path: xpackProject('plugin').path, configuration: 'testArtifacts')
 }
 
-Closure waitWithAuth = { NodeInfo node, AntBuilder ant ->
-    File tmpFile = new File(node.cwd, 'wait.success')
-    // wait up to twenty seconds
-    final long stopTime = System.currentTimeMillis() + 20000L;
-    Exception lastException = null;
-    while (System.currentTimeMillis() < stopTime) {
-        lastException = null;
-        // we use custom wait logic here as the elastic user is not available immediately and ant.get will fail when a 401 is returned
-        HttpURLConnection httpURLConnection = null;
-        try {
-            httpURLConnection = (HttpURLConnection) new URL("http://${node.httpUri()}/_cluster/health").openConnection();
-            httpURLConnection.setRequestProperty("Authorization", "Basic " +
-                    Base64.getEncoder().encodeToString("test_admin:x-pack-test-password".getBytes(StandardCharsets.UTF_8)));
-            httpURLConnection.setRequestMethod("GET");
-            httpURLConnection.setConnectTimeout(1000);
-            httpURLConnection.setReadTimeout(30000);
-            httpURLConnection.connect();
-            if (httpURLConnection.getResponseCode() == 200) {
-                tmpFile.withWriter StandardCharsets.UTF_8.name(), {
-                    it.write(httpURLConnection.getInputStream().getText(StandardCharsets.UTF_8.name()))
-                }
-                break;
-            }
-        } catch (Exception e) {
-            logger.debug("failed to call cluster health", e)
-            lastException = e
-        } finally {
-            if (httpURLConnection != null) {
-                httpURLConnection.disconnect();
-            }
-        }
-
-        // did not start, so wait a bit before trying again
-        Thread.sleep(500L);
-    }
-    if (tmpFile.exists() == false && lastException != null) {
-        logger.error("final attempt of calling cluster health failed", lastException)
-    }
-    return tmpFile.exists()
-}
-
 // copy xpack rest api
 File xpackResources = new File(xpackProject('plugin').projectDir, 'src/test/resources')
 project.copyRestSpec.from(xpackResources) {
   include 'rest-api-spec/api/**'
 }
-File jwks = new File(xpackProject('test:idp-fixture').projectDir, 'oidc/op-jwks.json')
-integTestCluster {
+
+testClusters.integTest {
+    extraConfigFile 'op-jwks.json', xpackProject('test:idp-fixture').file("oidc/op-jwks.json")
     setting 'xpack.security.enabled', 'true'
     setting 'xpack.security.authc.api_key.enabled', 'true'
     setting 'xpack.security.authc.token.enabled', 'true'
@@ -91,17 +51,13 @@ integTestCluster {
     setting 'xpack.security.authc.realms.oidc.oidc1.op.jwkset_path', 'op-jwks.json'
     setting 'xpack.security.authc.realms.oidc.oidc1.rp.redirect_uri', 'https://my.fantastic.rp/cb'
     setting 'xpack.security.authc.realms.oidc.oidc1.rp.client_id', 'elasticsearch-rp'
-    keystoreSetting 'xpack.security.authc.realms.oidc.oidc1.rp.client_secret', 'b07efb7a1cf6ec9462afe7b6d3ab55c6c7880262aa61ac28dded292aca47c9a2'
+    keystore 'xpack.security.authc.realms.oidc.oidc1.rp.client_secret', 'b07efb7a1cf6ec9462afe7b6d3ab55c6c7880262aa61ac28dded292aca47c9a2'
     setting 'xpack.security.authc.realms.oidc.oidc1.rp.response_type', 'id_token'
     setting 'xpack.security.authc.realms.oidc.oidc1.claims.principal', 'sub'
-    setupCommand 'setupTestAdmin',
-            'bin/elasticsearch-users', 'useradd', 'test_admin', '-p', 'x-pack-test-password', '-r', 'superuser'
-    waitCondition = waitWithAuth
-    extraConfigFile 'op-jwks.json', jwks
+    user username: 'test_admin'
 }
 
 
-
 buildRestTests.docs = fileTree(projectDir) {
     // No snippets in here!
     exclude 'build.gradle'