Test: Convert rolling upgrade test to have task per wire compat version (#24758)

This commit changes the rolling upgrade test to create a set of rest
test tasks per wire compat version. The most recent wire compat version
is always tested with the `integTest` task, and all versions can be
tested with `bwcTest`.

TESTING.asciidoc

@@ -449,6 +449,26 @@ Note: Starting vagrant VM outside of the elasticsearch folder requires to
 indicates the folder that contains the Vagrantfile using the VAGRANT_CWD
 environment variable.
 
+== Testing backwards compatibility
+
+Backwards compatibility tests exist to test upgrading from each supported version
+to the current version. To run all backcompat tests use:
+
+-------------------------------------------------
+gradle bwcTest
+-------------------------------------------------
+
+A specific version can be tested as well. For example, to test backcompat with
+version 5.3.2 run:
+
+-------------------------------------------------
+gradle v5.3.2#bwcTest
+-------------------------------------------------
+
+When running `gradle check`, some minimal backcompat checks are run. Which version
+is tested depends on the branch. On master, this will test against the current
+stable branch. On the stable branch, it will test against the latest release
+branch. Finally, on a release branch, it will test against the most recent release.
 
 == Coverage analysis
 

build.gradle

@@ -24,6 +24,7 @@ import org.eclipse.jgit.lib.RepositoryBuilder
 import org.gradle.plugins.ide.eclipse.model.SourceFolder
 import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.VersionProperties
+import org.elasticsearch.gradle.Version
 
 // common maven publishing configuration
 subprojects {
@@ -62,11 +63,11 @@ configure(subprojects.findAll { it.projectDir.toPath().startsWith(rootPath) }) {
 }
 
 // introspect all versions of ES that may be tested agains for backwards compatibility
-String currentVersion = VersionProperties.elasticsearch.minus('-SNAPSHOT')
-int prevMajor = Integer.parseInt(currentVersion.split('\\.')[0]) - 1
+Version currentVersion = Version.fromString(VersionProperties.elasticsearch.minus('-SNAPSHOT'))
+int prevMajor = currentVersion.major - 1
 File versionFile = file('core/src/main/java/org/elasticsearch/Version.java')
 List<String> versionLines = versionFile.readLines('UTF-8')
-List<String> versions = []
+List<Version> versions = []
 // keep track of the previous major version's last minor, so we know where wire compat begins
 int prevMinorIndex = -1 // index in the versions list of the last minor from the prev major
 int lastPrevMinor = -1 // the minor version number from the prev major we most recently seen
@@ -76,9 +77,9 @@ for (String line : versionLines) {
     int major = Integer.parseInt(match.group(1))
     int minor = Integer.parseInt(match.group(2))
     int bugfix = Integer.parseInt(match.group(3))
-    String versionStr = "${major}.${minor}.${bugfix}"
-    if (currentVersion != versionStr) {
-      versions.add(versionStr)
+    Version foundVersion = new Version(major, minor, bugfix, false)
+    if (currentVersion != foundVersion) {
+      versions.add(foundVersion)
     }
     if (major == prevMajor && minor > lastPrevMinor) {
       prevMinorIndex = versions.size() - 1
@@ -86,16 +87,17 @@ for (String line : versionLines) {
     }
   }
 }
-if (versions.toSorted() != versions) {
-  throw new GradleException('Versions.java contains out of order version constants')
+if (versions.toSorted { it.id } != versions) {
+  println "Versions: ${versions}" 
+  throw new GradleException("Versions.java contains out of order version constants")
 }
-if (currentVersion.split('\\.')[2].split('-')[0] == '0') {
+if (currentVersion.bugfix == 0) {
   // If on a release branch, after the initial release of that branch, the bugfix version will
   // be bumped, and will be != 0. On master and N.x branches, we want to test against the
   // unreleased version of closest branch. So for those cases, the version includes -SNAPSHOT,
-  // and the bwc-zip distribution will checkout and build that version. The version parsing
-  // logic above pulls the bugfix version, and then strips off any prerelease version
-  versions[-1] += '-SNAPSHOT'
+  // and the bwc-zip distribution will checkout and build that version.
+  Version last = versions[-1]
+  versions[-1] = new Version(last.major, last.minor, last.bugfix, true)
 }
 
 // injecting groovy property variables into all projects
@@ -154,7 +156,6 @@ subprojects {
     "org.elasticsearch.client:transport:${version}": ':client:transport',
     "org.elasticsearch.test:framework:${version}": ':test:framework',
     "org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:integ-test-zip',
-    "org.elasticsearch.distribution.zip:elasticsearch:${wireCompatVersions[-1]}": ':distribution:bwc-zip',
     "org.elasticsearch.distribution.zip:elasticsearch:${version}": ':distribution:zip',
     "org.elasticsearch.distribution.tar:elasticsearch:${version}": ':distribution:tar',
     "org.elasticsearch.distribution.rpm:elasticsearch:${version}": ':distribution:rpm',
@@ -167,6 +168,11 @@ subprojects {
     "org.elasticsearch.plugin:parent-join-client:${version}": ':modules:parent-join',
     "org.elasticsearch.plugin:percolator-client:${version}": ':modules:percolator',
   ]
+  if (wireCompatVersions[-1].snapshot) {
+    // if the most previous version is a snapshot, we need to connect that version to the
+    // bwc-zip project which will checkout and build that snapshot version
+    ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${wireCompatVersions[-1]}"] = ':distribution:bwc-zip'
+  }
   project.afterEvaluate {
     configurations.all {
       resolutionStrategy.dependencySubstitution { DependencySubstitutions subs ->

buildSrc/src/main/groovy/org/elasticsearch/gradle/Version.groovy

@@ -0,0 +1,77 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle
+
+/**
+ * Encapsulates comparison and printing logic for an x.y.z version.
+ */
+public class Version {
+
+    final int major
+    final int minor
+    final int bugfix
+    final int id
+    final boolean snapshot
+
+    public Version(int major, int minor, int bugfix, boolean snapshot) {
+        this.major = major
+        this.minor = minor
+        this.bugfix = bugfix
+        this.snapshot = snapshot
+        this.id = major * 100000 + minor * 1000 + bugfix * 10 + (snapshot ? 1 : 0)
+    }
+
+    public static Version fromString(String s) {
+        String[] parts = s.split('\\.')
+        String bugfix = parts[2]
+        boolean snapshot = false
+        if (bugfix.contains('-')) {
+            snapshot = bugfix.endsWith('-SNAPSHOT')
+            bugfix = bugfix.split('-')[0]
+        }
+        return new Version(parts[0] as int, parts[1] as int, bugfix as int, snapshot)
+    }
+
+    @Override
+    public String toString() {
+        String snapshotStr = snapshot ? '-SNAPSHOT' : ''
+        return "${major}.${minor}.${bugfix}${snapshotStr}"
+    }
+
+    public boolean equals(Version compareTo) {
+        return id == compareTo.id
+    }
+
+    public boolean before(String compareTo) {
+        return id < fromString(compareTo).id
+    }
+
+    public boolean onOrBefore(String compareTo) {
+        return id <= fromString(compareTo).id
+    }
+
+    public boolean onOrAfter(String compareTo) {
+        return id >= fromString(compareTo).id
+    }
+
+    public boolean after(String compareTo) {
+        return id > fromString(compareTo).id
+    }
+}

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -72,25 +72,25 @@ class ClusterFormationTasks {
             throw new GradleException("bwcVersion must not be null if numBwcNodes is > 0")
         }
         // this is our current version distribution configuration we use for all kinds of REST tests etc.
-        String distroConfigName = "${prefix}_elasticsearchDistro"
-        Configuration currentDistro = project.configurations.create(distroConfigName)
+        Configuration currentDistro = project.configurations.create("${prefix}_elasticsearchDistro")
+        Configuration bwcDistro = project.configurations.create("${prefix}_elasticsearchBwcDistro")
+        Configuration bwcPlugins = project.configurations.create("${prefix}_elasticsearchBwcPlugins")
         configureDistributionDependency(project, config.distribution, currentDistro, VersionProperties.elasticsearch)
-        if (config.bwcVersion != null && config.numBwcNodes > 0) {
+        if (config.numBwcNodes > 0) {
+            if (config.bwcVersion == null) {
+                throw new IllegalArgumentException("Must specify bwcVersion when numBwcNodes > 0")
+            }
             // if we have a cluster that has a BWC cluster we also need to configure a dependency on the BWC version
             // this version uses the same distribution etc. and only differs in the version we depend on.
             // from here on everything else works the same as if it's the current version, we fetch the BWC version
             // from mirrors using gradles built-in mechanism etc.
-            project.configurations {
-                elasticsearchBwcDistro
-                elasticsearchBwcPlugins
-            }
-            configureDistributionDependency(project, config.distribution, project.configurations.elasticsearchBwcDistro, config.bwcVersion)
+
+            configureDistributionDependency(project, config.distribution, bwcDistro, config.bwcVersion)
             for (Map.Entry<String, Project> entry : config.plugins.entrySet()) {
-                configureBwcPluginDependency("${prefix}_elasticsearchBwcPlugins", project, entry.getValue(),
-                        project.configurations.elasticsearchBwcPlugins, config.bwcVersion)
+                configureBwcPluginDependency("${prefix}_elasticsearchBwcPlugins", project, entry.getValue(), bwcPlugins, config.bwcVersion)
             }
-            project.configurations.elasticsearchBwcDistro.resolutionStrategy.cacheChangingModulesFor(0, TimeUnit.SECONDS)
-            project.configurations.elasticsearchBwcPlugins.resolutionStrategy.cacheChangingModulesFor(0, TimeUnit.SECONDS)
+            bwcDistro.resolutionStrategy.cacheChangingModulesFor(0, TimeUnit.SECONDS)
+            bwcPlugins.resolutionStrategy.cacheChangingModulesFor(0, TimeUnit.SECONDS)
         }
         for (int i = 0; i < config.numNodes; i++) {
             // we start N nodes and out of these N nodes there might be M bwc nodes.
@@ -99,7 +99,7 @@ class ClusterFormationTasks {
             Configuration distro = currentDistro
             if (i < config.numBwcNodes) {
                 elasticsearchVersion = config.bwcVersion
-                distro = project.configurations.elasticsearchBwcDistro
+                distro = bwcDistro
             }
             NodeInfo node = new NodeInfo(config, i, project, prefix, elasticsearchVersion, sharedDir)
             nodes.add(node)
@@ -171,7 +171,7 @@ class ClusterFormationTasks {
             if (node.nodeVersion == VersionProperties.elasticsearch) {
                 setup = configureCopyPluginsTask(taskName(prefix, node, 'copyPlugins'), project, setup, node)
             } else {
-                setup = configureCopyBwcPluginsTask(taskName(prefix, node, 'copyBwcPlugins'), project, setup, node)
+                setup = configureCopyBwcPluginsTask(taskName(prefix, node, 'copyBwcPlugins'), project, setup, node, prefix)
             }
         }
 
@@ -417,7 +417,8 @@ class ClusterFormationTasks {
     }
 
     /** Configures task to copy a plugin based on a zip file resolved using dependencies for an older version */
-    static Task configureCopyBwcPluginsTask(String name, Project project, Task setup, NodeInfo node) {
+    static Task configureCopyBwcPluginsTask(String name, Project project, Task setup, NodeInfo node, String prefix) {
+        Configuration bwcPlugins = project.configurations.getByName("${prefix}_elasticsearchBwcPlugins")
         for (Map.Entry<String, Project> plugin : node.config.plugins.entrySet()) {
             Project pluginProject = plugin.getValue()
             verifyProjectHasBuildPlugin(name, node.nodeVersion, project, pluginProject)
@@ -428,14 +429,15 @@ class ClusterFormationTasks {
             }
 
             final String depName = pluginProject.extensions.findByName('esplugin').name
-            Dependency dep = project.configurations.elasticsearchBwcPlugins.dependencies.find {
+
+            Dependency dep = bwcPlugins.dependencies.find {
                 it.name == depName
             }
             configuration.dependencies.add(dep)
         }
 
         Copy copyPlugins = project.tasks.create(name: name, type: Copy, dependsOn: setup) {
-            from project.configurations.elasticsearchBwcPlugins
+            from bwcPlugins
             into node.pluginsTmpDir
         }
         return copyPlugins

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy

@@ -51,8 +51,6 @@ public class RestIntegTestTask extends DefaultTask {
     boolean includePackaged = false
 
     public RestIntegTestTask() {
-        description = 'Runs rest tests against an elasticsearch cluster.'
-        group = JavaBasePlugin.VERIFICATION_GROUP
         runner = project.tasks.create("${name}Runner", RandomizedTestingTask.class)
         super.dependsOn(runner)
         clusterInit = project.tasks.create(name: "${name}Cluster#init", dependsOn: project.testClasses)

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestTestPlugin.groovy

@@ -22,6 +22,7 @@ import org.elasticsearch.gradle.BuildPlugin
 import org.gradle.api.InvalidUserDataException
 import org.gradle.api.Plugin
 import org.gradle.api.Project
+import org.gradle.api.plugins.JavaBasePlugin
 
 /**
  * Adds support for starting an Elasticsearch cluster before running integration
@@ -43,6 +44,8 @@ public class RestTestPlugin implements Plugin<Project> {
         }
 
         RestIntegTestTask integTest = project.tasks.create('integTest', RestIntegTestTask.class)
+        integTest.description = 'Runs rest tests against an elasticsearch cluster.'
+        integTest.group = JavaBasePlugin.VERIFICATION_GROUP
         integTest.clusterConfig.distribution = 'zip' // rest tests should run with the real zip
         integTest.mustRunAfter(project.precommit)
         project.check.dependsOn(integTest)

qa/rolling-upgrade/build.gradle

@@ -18,71 +18,90 @@
  */
 
 import org.elasticsearch.gradle.test.RestIntegTestTask
+import org.elasticsearch.gradle.Version
 
 apply plugin: 'elasticsearch.standalone-test'
 
-task oldClusterTest(type: RestIntegTestTask) {
-  mustRunAfter(precommit)
+// This is a top level task which we will add dependencies to below.
+// It is a single task that can be used to backcompat tests against all versions.
+task bwcTest {
+  description = 'Runs backwards compatibility tests.'
+  group = 'verification'
 }
 
-oldClusterTestCluster {
-  distribution = 'zip'
-  bwcVersion = project.wireCompatVersions[-1] // TODO: either randomize, or make this settable with sysprop
-  numBwcNodes = 2
-  numNodes = 2
-  clusterName = 'rolling-upgrade'
-  setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
-  setting 'http.content_type.required', 'true'
-}
+for (Version version : wireCompatVersions) {
+  String baseName = "v${version}"
 
-oldClusterTestRunner {
-  systemProperty 'tests.rest.suite', 'old_cluster'
-}
+  Task oldClusterTest = tasks.create(name: "${baseName}#oldClusterTest", type: RestIntegTestTask) {
+    mustRunAfter(precommit)
+  }
 
-task mixedClusterTest(type: RestIntegTestTask) {}
+  Object extension = extensions.findByName("${baseName}#oldClusterTestCluster")
+  configure(extensions.findByName("${baseName}#oldClusterTestCluster")) {
+    distribution = 'zip'
+    bwcVersion = version
+    numBwcNodes = 2
+    numNodes = 2
+    clusterName = 'rolling-upgrade'
+    setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
+    if (version.onOrAfter('5.3.0')) {
+      setting 'http.content_type.required', 'true'
+    }
+  }
 
-mixedClusterTestCluster {
-  dependsOn oldClusterTestRunner, 'oldClusterTestCluster#node1.stop'
-  distribution = 'zip'
-  clusterName = 'rolling-upgrade'
-  unicastTransportUri = { seedNode, node, ant -> oldClusterTest.nodes.get(0).transportUri() }
-  dataDir = "${-> oldClusterTest.nodes[1].dataDir}"
-  setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
-}
+  Task oldClusterTestRunner = tasks.getByName("${baseName}#oldClusterTestRunner")
+  oldClusterTestRunner.configure {
+    systemProperty 'tests.rest.suite', 'old_cluster'
+  }
 
-mixedClusterTestRunner {
-  systemProperty 'tests.rest.suite', 'mixed_cluster'
-  finalizedBy 'oldClusterTestCluster#node0.stop'
-}
+  Task mixedClusterTest = tasks.create(name: "${baseName}#mixedClusterTest", type: RestIntegTestTask)
 
-task upgradedClusterTest(type: RestIntegTestTask) {
-  dependsOn(mixedClusterTestRunner, 'oldClusterTestCluster#node0.stop')
-}
+  configure(extensions.findByName("${baseName}#mixedClusterTestCluster")) {
+    dependsOn oldClusterTestRunner, "${baseName}#oldClusterTestCluster#node1.stop"
+    distribution = 'zip'
+    clusterName = 'rolling-upgrade'
+    unicastTransportUri = { seedNode, node, ant -> oldClusterTest.nodes.get(0).transportUri() }
+    dataDir = "${-> oldClusterTest.nodes[1].dataDir}"
+    setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
+  }
 
-upgradedClusterTestCluster {
-  distribution = 'zip'
-  clusterName = 'rolling-upgrade'
-  unicastTransportUri = { seedNode, node, ant -> mixedClusterTest.nodes.get(0).transportUri() }
-  dataDir = "${-> oldClusterTest.nodes[0].dataDir}"
-  setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
-}
+  Task mixedClusterTestRunner = tasks.getByName("${baseName}#mixedClusterTestRunner")
+  mixedClusterTestRunner.configure {
+    systemProperty 'tests.rest.suite', 'mixed_cluster'
+    finalizedBy "${baseName}#oldClusterTestCluster#node0.stop"
+  }
 
-upgradedClusterTestRunner {
-  systemProperty 'tests.rest.suite', 'upgraded_cluster'
-  // only need to kill the mixed cluster tests node here because we explicitly told it to not stop nodes upon completion
-  finalizedBy 'mixedClusterTestCluster#stop'
-}
+  Task upgradedClusterTest = tasks.create(name: "${baseName}#upgradedClusterTest", type: RestIntegTestTask) {
+    dependsOn(mixedClusterTestRunner, "${baseName}#oldClusterTestCluster#node0.stop")
+  }
 
-task integTest {
-  dependsOn = [upgradedClusterTest]
+  configure(extensions.findByName("${baseName}#upgradedClusterTestCluster")) {
+    distribution = 'zip'
+    clusterName = 'rolling-upgrade'
+    unicastTransportUri = { seedNode, node, ant -> mixedClusterTest.nodes.get(0).transportUri() }
+    dataDir = "${-> oldClusterTest.nodes[0].dataDir}"
+    setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
+  }
+
+  Task upgradedClusterTestRunner = tasks.getByName("${baseName}#upgradedClusterTestRunner")
+  upgradedClusterTestRunner.configure {
+    systemProperty 'tests.rest.suite', 'upgraded_cluster'
+    // only need to kill the mixed cluster tests node here because we explicitly told it to not stop nodes upon completion
+    finalizedBy "${baseName}#mixedClusterTestCluster#stop"
+  }
+
+  Task versionBwcTest = tasks.create(name: "${baseName}#bwcTest") {
+    dependsOn = [upgradedClusterTest]
+  }
+
+  bwcTest.dependsOn(versionBwcTest)
 }
 
 test.enabled = false // no unit tests for rolling upgrades, only the rest integration test
 
-check.dependsOn(integTest)
-
-repositories {
-  maven {
-    url "https://oss.sonatype.org/content/repositories/snapshots/"
-  }
+// basic integ tests includes testing bwc against the most recent version
+task integTest {
+  dependsOn = ["v${wireCompatVersions[-1]}#bwcTest"]
 }
+
+check.dependsOn(integTest)