Testclusters fix bwc (#46740)

Additions to make testclsuters work with lather versions of ES

buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java

@@ -202,15 +202,22 @@ public class DistributionDownloadPlugin implements Plugin<Project> {
         }
 
         String extension = distribution.getType().toString();
-        String classifier = "x86_64";
+        String classifier = ":x86_64";
         if (distribution.getType() == Type.ARCHIVE) {
             extension = distribution.getPlatform() == Platform.WINDOWS ? "zip" : "tar.gz";
-            classifier = distribution.getPlatform() + "-" + classifier;
+            if (distribution.getVersion().onOrAfter("7.0.0")) {
+                classifier = ":" + distribution.getPlatform() + "-x86_64";
+            } else {
+                classifier = "";
+            }
         } else if (distribution.getType() == Type.DEB) {
-            classifier = "amd64";
+            classifier = ":amd64";
+        }
+        String flavor = "";
+        if (distribution.getFlavor() == Flavor.OSS && distribution.getVersion().onOrAfter("6.3.0")) {
+            flavor = "-oss";
         }
-        return FAKE_IVY_GROUP + ":elasticsearch" + (distribution.getFlavor() == Flavor.OSS ? "-oss:" : ":")
-            + distribution.getVersion() + ":" + classifier + "@" + extension;
+        return FAKE_IVY_GROUP + ":elasticsearch" + flavor + ":" + distribution.getVersion() + classifier + "@" + extension;
     }
 
     private static Dependency projectDependency(Project project, String projectPath, String projectConfig) {

buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java

@@ -161,7 +161,7 @@ public class ElasticsearchDistribution implements Buildable, Iterable<File> {
         return bundledJdk.getOrElse(true);
     }
 
-    public void setBundledJdk(boolean bundledJdk) {
+    public void setBundledJdk(Boolean bundledJdk) {
         this.bundledJdk.set(bundledJdk);
     }
 
@@ -197,15 +197,15 @@ public class ElasticsearchDistribution implements Buildable, Iterable<File> {
     void finalizeValues() {
 
         if (getType() == Type.INTEG_TEST_ZIP) {
-            if (platform.isPresent()) {
+            if (platform.getOrNull() != null) {
                 throw new IllegalArgumentException(
                     "platform not allowed for elasticsearch distribution [" + name + "] of type [integ_test_zip]");
             }
-            if (flavor.isPresent()) {
+            if (flavor.getOrNull() != null) {
                 throw new IllegalArgumentException(
-                    "flavor not allowed for elasticsearch distribution [" + name + "] of type [integ_test_zip]");
+                    "flavor [" + flavor.get() + "] not allowed for elasticsearch distribution [" + name + "] of type [integ_test_zip]");
             }
-            if (bundledJdk.isPresent()) {
+            if (bundledJdk.getOrNull() != null) {
                 throw new IllegalArgumentException(
                     "bundledJdk not allowed for elasticsearch distribution [" + name + "] of type [integ_test_zip]");
             }

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java

@@ -254,16 +254,36 @@ public class ElasticsearchCluster implements TestClusterConfiguration, Named {
         } else {
             nodeNames = nodes.stream().map(ElasticsearchNode::getName).map(this::safeName).collect(Collectors.joining(","));
         }
+        ElasticsearchNode firstNode = null;
         for (ElasticsearchNode node : nodes) {
+            // Can only configure master nodes if we have node names defined
             if (nodeNames != null) {
-                // Can only configure master nodes if we have node names defined
-                if (node.getVersion().getMajor() >= 7) {
+                if (node.getVersion().onOrAfter("7.0.0")) {
                     node.defaultConfig.put("cluster.initial_master_nodes", "[" + nodeNames + "]");
                     node.defaultConfig.put("discovery.seed_providers", "file");
                     node.defaultConfig.put("discovery.seed_hosts", "[]");
+                } else {
+                    node.defaultConfig.put("discovery.zen.master_election.wait_for_joins_timeout", "5s");
+                    if (nodes.size() > 1) {
+                        node.defaultConfig.put("discovery.zen.minimum_master_nodes", Integer.toString(nodes.size() / 2 + 1));
+                    }
+                    if (node.getVersion().onOrAfter("6.5.0")) {
+                        node.defaultConfig.put("discovery.zen.hosts_provider", "file");
+                        node.defaultConfig.put("discovery.zen.ping.unicast.hosts", "[]");
+                    } else {
+                        if (firstNode == null) {
+                            node.defaultConfig.put("discovery.zen.ping.unicast.hosts", "[]");
+                        } else {
+                            firstNode.waitForAllConditions();
+                            node.defaultConfig.put("discovery.zen.ping.unicast.hosts", "[\"" + firstNode.getTransportPortURI() + "\"]");
+                        }
+                    }
                 }
             }
             node.start();
+            if (firstNode == null) {
+                firstNode = node;
+            }
         }
     }
 

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -185,25 +185,30 @@ public class ElasticsearchNode implements TestClusterConfiguration {
     @Override
     public void setVersion(String version) {
         requireNonNull(version, "null version passed when configuring test cluster `" + this + "`");
-        String distroName = "testclusters" + path.replace(":", "-") + "-" + this.name + "-" + version + "-";
-        NamedDomainObjectContainer<ElasticsearchDistribution> container = DistributionDownloadPlugin.getContainer(project);
-        if (container.findByName(distroName) == null){
-            container.create(distroName);
-        }
-        ElasticsearchDistribution distro = container.getByName(distroName);
-        distro.setVersion(version);
-        setDistributionType(distro, testDistribution);
-        distributions.add(distro);
+        checkFrozen();
+        distributions.clear();
+        doSetVersion(version);
     }
 
     @Override
     public void setVersions(List<String> versions) {
         requireNonNull(versions, "null version list passed when configuring test cluster `" + this + "`");
-        checkFrozen();
         distributions.clear();
         for (String version : versions) {
-            setVersion(version);
+            doSetVersion(version);
+        }
+    }
+
+    private void doSetVersion(String version) {
+        String distroName = "testclusters" + path.replace(":", "-") + "-" + this.name + "-" + version + "-";
+        NamedDomainObjectContainer<ElasticsearchDistribution> container = DistributionDownloadPlugin.getContainer(project);
+        if (container.findByName(distroName) == null) {
+            container.create(distroName);
         }
+        ElasticsearchDistribution distro = container.getByName(distroName);
+        distro.setVersion(version);
+        setDistributionType(distro, testDistribution);
+        distributions.add(distro);
     }
 
     @Internal
@@ -230,6 +235,10 @@ public class ElasticsearchNode implements TestClusterConfiguration {
     private void setDistributionType(ElasticsearchDistribution distribution, TestDistribution testDistribution) {
         if (testDistribution == TestDistribution.INTEG_TEST) {
             distribution.setType(ElasticsearchDistribution.Type.INTEG_TEST_ZIP);
+            // we change the underlying distribution when changing the test distribution of the cluster.
+            distribution.setFlavor(null);
+            distribution.setPlatform(null);
+            distribution.setBundledJdk(null);
         } else {
             distribution.setType(ElasticsearchDistribution.Type.ARCHIVE);
             if (testDistribution == TestDistribution.DEFAULT) {
@@ -408,6 +417,14 @@ public class ElasticsearchNode implements TestClusterConfiguration {
             );
         }
 
+        if (getVersion().before("6.3.0") && testDistribution == TestDistribution.DEFAULT) {
+            LOGGER.info("emulating the {} flavor for {} by installing x-pack", testDistribution, getVersion());
+            runElaticsearchBinScript(
+                "elasticsearch-plugin",
+                "install", "--batch", "x-pack"
+            );
+        }
+
         if (keystoreSettings.isEmpty() == false || keystoreFiles.isEmpty() == false) {
             logToProcessStdout("Adding " + keystoreSettings.size() + " keystore settings and " + keystoreFiles.size() + " keystore files");
             runElaticsearchBinScript("elasticsearch-keystore", "create");
@@ -430,13 +447,17 @@ public class ElasticsearchNode implements TestClusterConfiguration {
 
         copyExtraConfigFiles();
 
-        if (isSettingMissingOrTrue("xpack.security.enabled")) {
-            logToProcessStdout("Setting up " + credentials.size() + " users");
+        if (isSettingTrue("xpack.security.enabled")) {
             if (credentials.isEmpty()) {
                 user(Collections.emptyMap());
             }
+        }
+
+        if (credentials.isEmpty() == false) {
+            logToProcessStdout("Setting up " + credentials.size() + " users");
+
             credentials.forEach(paramMap -> runElaticsearchBinScript(
-                "elasticsearch-users",
+                getVersion().onOrAfter("6.3.0") ? "elasticsearch-users" : "x-pack/users",
                 paramMap.entrySet().stream()
                     .flatMap(entry -> Stream.of(entry.getKey(), entry.getValue()))
                     .toArray(String[]::new)
@@ -481,7 +502,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
         start();
     }
 
-    private boolean isSettingMissingOrTrue(String name) {
+    private boolean isSettingTrue(String name) {
         return Boolean.valueOf(settings.getOrDefault(name, "false").toString());
     }
 

buildSrc/src/test/java/org/elasticsearch/gradle/DistributionDownloadPluginIT.java

@@ -56,6 +56,11 @@ public class DistributionDownloadPluginIT extends GradleIntegrationTestCase {
     }
 
     public void testReleased() throws Exception {
+        doTestReleased("7.0.0", "/downloads/elasticsearch/elasticsearch-7.0.0-windows-x86_64.zip");
+        doTestReleased("6.5.0", "/downloads/elasticsearch/elasticsearch-6.5.0.zip");
+    }
+
+    private void doTestReleased(String version, String urlPath) throws IOException {
         WireMockServer wireMock = new WireMockServer(0);
         try {
             final byte[] filebytes;
@@ -63,12 +68,11 @@ public class DistributionDownloadPluginIT extends GradleIntegrationTestCase {
                      Files.newInputStream(Paths.get("src/testKit/distribution-download/distribution/files/fake_elasticsearch.zip"))) {
                 filebytes = stream.readAllBytes();
             }
-            String urlPath = "/downloads/elasticsearch/elasticsearch-1.0.0-windows-x86_64.zip";
             wireMock.stubFor(head(urlEqualTo(urlPath)).willReturn(aResponse().withStatus(200)));
             wireMock.stubFor(get(urlEqualTo(urlPath)).willReturn(aResponse().withStatus(200).withBody(filebytes)));
             wireMock.start();
 
-            assertExtractedDistro("1.0.0", "archive", "windows", null, null,
+            assertExtractedDistro(version, "archive", "windows", null, null,
                 "tests.download_service", wireMock.baseUrl());
         } catch (Exception e) {
             // for debugging

buildSrc/src/test/java/org/elasticsearch/gradle/DistributionDownloadPluginTests.java

@@ -90,7 +90,7 @@ public class DistributionDownloadPluginTests extends GradleUnitTestCase {
     public void testFlavorForIntegTest() {
         assertDistroError(createProject(null),
             "testdistro", "5.0.0", Type.INTEG_TEST_ZIP, null, Flavor.OSS, null,
-            "flavor not allowed for elasticsearch distribution [testdistro]");
+            "flavor [oss] not allowed for elasticsearch distribution [testdistro] of type [integ_test_zip]");
     }
 
     public void testBundledJdkDefault() {