Simplify integ test distribution types (#37618)

The integ tests currently use the raw zip project name as the
distribution type. This commit simplifies this specification to be
"default" or "oss". Whether zip or tar is used should be an internal
implementation detail of the integ test setup, which can (in the future)
be platform specific.

TESTING.asciidoc

@@ -42,7 +42,7 @@ In order to start with a different distribution use the `-Drun.distribution` arg
 To for example start the open source distribution:
 
 -------------------------------------
-./gradlew run -Drun.distribution=oss-zip
+./gradlew run -Drun.distribution=oss
 -------------------------------------
 
 ==== License type

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy

@@ -34,7 +34,7 @@ public class DocsTestPlugin extends RestTestPlugin {
         project.pluginManager.apply('elasticsearch.standalone-rest-test')
         super.apply(project)
         // The distribution can be configured with -Dtests.distribution on the command line
-        project.integTestCluster.distribution = System.getProperty('tests.distribution', 'zip')
+        project.integTestCluster.distribution = System.getProperty('tests.distribution', 'default')
         // Docs are published separately so no need to assemble
         project.tasks.assemble.enabled = false
         Map<String, String> defaultSubstitutions = [

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy

@@ -29,7 +29,7 @@ class ClusterConfiguration {
     private final Project project
 
     @Input
-    String distribution = 'zip'
+    String distribution = 'default'
 
     @Input
     int numNodes = 1

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -29,7 +29,6 @@ import org.elasticsearch.gradle.plugin.PluginPropertiesExtension
 import org.gradle.api.AntBuilder
 import org.gradle.api.DefaultTask
 import org.gradle.api.GradleException
-import org.gradle.api.InvalidUserDataException
 import org.gradle.api.Project
 import org.gradle.api.Task
 import org.gradle.api.artifacts.Configuration
@@ -87,7 +86,7 @@ class ClusterFormationTasks {
         Configuration currentDistro = project.configurations.create("${prefix}_elasticsearchDistro")
         Configuration bwcDistro = project.configurations.create("${prefix}_elasticsearchBwcDistro")
         Configuration bwcPlugins = project.configurations.create("${prefix}_elasticsearchBwcPlugins")
-        if (System.getProperty('tests.distribution', 'oss-zip') == 'integ-test-zip') {
+        if (System.getProperty('tests.distribution', 'oss') == 'integ-test-zip') {
             throw new Exception("tests.distribution=integ-test-zip is not supported")
         }
         configureDistributionDependency(project, config.distribution, currentDistro, VersionProperties.elasticsearch)
@@ -172,31 +171,31 @@ class ClusterFormationTasks {
 
     /** Adds a dependency on the given distribution */
     static void configureDistributionDependency(Project project, String distro, Configuration configuration, String elasticsearchVersion) {
+        // TEMP HACK
+        // The oss docs CI build overrides the distro on the command line. This hack handles backcompat until CI is updated.
+        if (distro.equals('oss-zip')) {
+            distro = 'oss'
+        }
+        if (distro.equals('zip')) {
+            distro = 'default'
+        }
+        // END TEMP HACK
+        if (['integ-test-zip', 'oss', 'default'].contains(distro) == false) {
+            throw new GradleException("Unknown distribution: ${distro} in project ${project.path}")
+        }
         Version version = Version.fromString(elasticsearchVersion)
         if (version.before('6.3.0') && distro.startsWith('oss-')) {
             distro = distro.substring('oss-'.length())
         }
-        String os = "linux"
-        if (Os.FAMILY_WINDOWS) {
-            os = "windows"
-        } else if (Os.FAMILY_MAC) {
-            os = "darwin"
-        }
-        String packaging = distro
-        if (distro.contains('tar')) {
-            packaging = 'tar.gz'\
-        } else if (distro.contains('zip')) {
-            packaging = 'zip'
-        }
-        String group = "downloads.${packaging}"
+        String group = "downloads.zip"
         if (distro.equals("integ-test-zip")) {
             group = "org.elasticsearch.distribution.integ-test-zip"
         }
         String artifactName = 'elasticsearch'
-        if (distro.contains('oss')) {
+        if (distro.equals('oss') && Version.fromString(elasticsearchVersion).onOrAfter('6.3.0')) {
             artifactName += '-oss'
         }
-        project.dependencies.add(configuration.name, "${group}:${artifactName}:${elasticsearchVersion}@${packaging}")
+        project.dependencies.add(configuration.name, "${group}:${artifactName}:${elasticsearchVersion}@zip")
     }
 
     /** Adds a dependency on a different version of the given plugin, which will be retrieved using gradle's dependency resolution */
@@ -319,31 +318,13 @@ class ClusterFormationTasks {
           elasticsearch source tree. If this is a plugin built in the elasticsearch source tree or this is a distro in
           the elasticsearch source tree then this should be the version of elasticsearch built by the source tree.
           If it isn't then Bad Things(TM) will happen. */
-        Task extract
-
-        switch (node.config.distribution) {
-            case 'integ-test-zip':
-            case 'zip':
-            case 'oss-zip':
-                extract = project.tasks.create(name: name, type: Copy, dependsOn: extractDependsOn) {
-                    from {
-                        project.zipTree(configuration.singleFile)
-                    }
-                    into node.baseDir
-                }
-                break;
-            case 'tar':
-            case 'oss-tar':
-                extract = project.tasks.create(name: name, type: Copy, dependsOn: extractDependsOn) {
-                    from {
-                        project.tarTree(project.resources.gzip(configuration.singleFile))
-                    }
-                    into node.baseDir
-                }
-                break;
-            default:
-                throw new InvalidUserDataException("Unknown distribution: ${node.config.distribution}")
+        Task extract = project.tasks.create(name: name, type: Copy, dependsOn: extractDependsOn) {
+            from {
+                project.zipTree(configuration.singleFile)
+            }
+            into node.baseDir
         }
+
         return extract
     }
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy

@@ -29,9 +29,6 @@ import org.gradle.api.Project
 import java.nio.file.Files
 import java.nio.file.Path
 import java.nio.file.Paths
-
-import static org.elasticsearch.gradle.BuildPlugin.getJavaHome
-
 /**
  * A container for the files and configuration associated with a single node in a test cluster.
  */
@@ -125,8 +122,8 @@ class NodeInfo {
         baseDir = new File(project.buildDir, "cluster/${prefix} node${nodeNum}")
         pidFile = new File(baseDir, 'es.pid')
         this.nodeVersion = Version.fromString(nodeVersion)
-        homeDir = homeDir(baseDir, config.distribution, nodeVersion)
-        pathConf = pathConf(baseDir, config.distribution, nodeVersion)
+        homeDir = new File(baseDir, "elasticsearch-${nodeVersion}")
+        pathConf = new File(homeDir, 'config')
         if (config.dataDir != null) {
             dataDir = "${config.dataDir(nodeNum)}"
         } else {
@@ -299,41 +296,4 @@ class NodeInfo {
         }
         return dataDir
     }
-
-    /** Returns the directory elasticsearch home is contained in for the given distribution */
-    static File homeDir(File baseDir, String distro, String nodeVersion) {
-        String path
-        switch (distro) {
-            case 'integ-test-zip':
-            case 'zip':
-            case 'tar':
-            case 'oss-zip':
-            case 'oss-tar':
-                path = "elasticsearch-${nodeVersion}"
-                break
-            case 'rpm':
-            case 'deb':
-                path = "${distro}-extracted/usr/share/elasticsearch"
-                break
-            default:
-                throw new InvalidUserDataException("Unknown distribution: ${distro}")
-        }
-        return new File(baseDir, path)
-    }
-
-    static File pathConf(File baseDir, String distro, String nodeVersion) {
-        switch (distro) {
-            case 'integ-test-zip':
-            case 'zip':
-            case 'oss-zip':
-            case 'tar':
-            case 'oss-tar':
-                return new File(homeDir(baseDir, distro, nodeVersion), 'config')
-            case 'rpm':
-            case 'deb':
-                return new File(baseDir, "${distro}-extracted/etc/elasticsearch")
-            default:
-                throw new InvalidUserDataException("Unknown distribution: ${distro}")
-        }
-    }
 }

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RunTask.groovy

@@ -16,7 +16,7 @@ public class RunTask extends DefaultTask {
         clusterConfig.httpPort = 9200
         clusterConfig.transportPort = 9300
         clusterConfig.daemonize = false
-        clusterConfig.distribution = 'zip'
+        clusterConfig.distribution = 'default'
         project.afterEvaluate {
             ClusterFormationTasks.setup(project, name, this, clusterConfig)
         }

distribution/build.gradle

@@ -337,8 +337,8 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
 }
 
 task run(type: RunTask) {
-  distribution = System.getProperty('run.distribution', 'zip')
-  if (distribution == 'zip') {
+  distribution = System.getProperty('run.distribution', 'default')
+  if (distribution == 'default') {
     String licenseType = System.getProperty("run.license_type", "basic")
     if (licenseType == 'trial') {
       setting 'xpack.ml.enabled', 'true'

docs/build.gradle

@@ -36,7 +36,7 @@ buildRestTests.expectedUnconvertedCandidates = [
 ]
 
 integTestCluster {
-  if ("zip".equals(integTestCluster.distribution)) {
+  if ("default".equals(integTestCluster.distribution)) {
     setting 'xpack.license.self_generated.type', 'trial'
   }
 

plugins/examples/painless-whitelist/build.gradle

@@ -32,7 +32,7 @@ dependencies {
 }
 
 if (System.getProperty('tests.distribution') == null) {
-  integTestCluster.distribution = 'oss-zip'
+  integTestCluster.distribution = 'oss'
 }
 
 unitTest.enabled = false

qa/build.gradle

@@ -4,8 +4,8 @@ import org.elasticsearch.gradle.test.RestIntegTestTask
 subprojects { Project subproj ->
   subproj.tasks.withType(RestIntegTestTask) {
     subproj.extensions.configure("${it.name}Cluster") { cluster ->
-      cluster.distribution = System.getProperty('tests.distribution', 'oss-zip')
-      if (cluster.distribution == 'zip') {
+      cluster.distribution = System.getProperty('tests.distribution', 'oss')
+      if (cluster.distribution == 'default') {
         /*
          * Add Elastic's repositories so we can resolve older versions of the
          * default distribution. Those aren't in maven central.

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/XPackIT.java

@@ -30,13 +30,13 @@ import static org.junit.Assume.assumeThat;
 
 /**
  * Basic tests for simple xpack functionality that are only run if the
- * cluster is the on the "zip" distribution.
+ * cluster is the on the default distribution.
  */
 public class XPackIT extends AbstractRollingTestCase {
     @Before
     public void skipIfNotXPack() {
         assumeThat("test is only supported if the distribution contains xpack",
-                System.getProperty("tests.distribution"), equalTo("zip"));
+                System.getProperty("tests.distribution"), equalTo("default"));
         assumeThat("running this on the unupgraded cluster would change its state and it wouldn't work prior to 6.3 anyway",
                 CLUSTER_TYPE, equalTo(ClusterType.UPGRADED));
         /*

qa/smoke-test-multinode/build.gradle

@@ -29,7 +29,7 @@ integTestCluster {
 }
 
 integTestRunner {
-  if ('zip'.equals(integTestCluster.distribution)) {
+  if ('default'.equals(integTestCluster.distribution)) {
     systemProperty 'tests.rest.blacklist', [
         'cat.templates/10_basic/No templates',
         'cat.templates/10_basic/Sort templates',

x-pack/plugin/build.gradle

@@ -141,7 +141,7 @@ integTestCluster {
   setting 'xpack.license.self_generated.type', 'trial'
   keystoreSetting 'bootstrap.password', 'x-pack-test-password'
   keystoreSetting 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode'
-  distribution = 'zip' // this is important since we use the reindex module in ML
+  distribution = 'default' // this is important since we use the reindex module in ML
 
   setupCommand 'setupTestUser', 'bin/elasticsearch-users', 'useradd', 'x_pack_rest_user', '-p', 'x-pack-test-password', '-r', 'superuser'
 

x-pack/plugin/ccr/qa/rest/build.gradle

@@ -12,7 +12,7 @@ task restTest(type: RestIntegTestTask) {
 }
 
 restTestCluster {
-    distribution 'zip'
+    distribution 'default'
     // Disable assertions in FollowingEngineAssertions, otherwise an AssertionError is thrown before
     // indexing a document directly in a follower index. In a rest test we like to test the exception
     // that is thrown in production when indexing a document directly in a follower index.

x-pack/plugin/ilm/qa/rest/build.gradle

@@ -21,7 +21,7 @@ restTestRunner {
 }
 
 restTestCluster {
-    distribution 'zip'
+    distribution 'default'
     setting 'xpack.ilm.enabled', 'true'
     setting 'xpack.ml.enabled', 'false'
     setting 'xpack.monitoring.enabled', 'false'

x-pack/qa/audit-tests/build.gradle

@@ -16,7 +16,7 @@ task copyXPackPluginProps(type: Copy) { // wth is this?
 project.sourceSets.test.output.dir(outputDir, builtBy: copyXPackPluginProps)
 
 integTestCluster {
-  distribution 'zip'
+  distribution 'default'
   setting 'xpack.ilm.enabled', 'false'
   setting 'xpack.ml.enabled', 'false'
   setting 'xpack.monitoring.enabled', 'false'

x-pack/qa/security-example-spi-extension/build.gradle

@@ -32,7 +32,7 @@ integTestCluster {
   // This is important, so that all the modules are available too.
   // There are index templates that use token filters that are in analysis-module and
   // processors are being used that are in ingest-common module.
-  distribution = 'zip'
+  distribution = 'default'
 
   setupCommand 'setupDummyUser',
                'bin/elasticsearch-users', 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser'