Generalize BWC logic (#28505)

Generalizing BWC building so that there is less code to modify for a release. This ensures we do not
need to think about what major or minor version is in the gradle code. It follows the general rules of the
elastic release structure. For more information on the rules, see the VersionCollection's javadoc.

This also removes the additional bwc snapshots that will never be released, such as 6.0.2, which were
being built and tested against every time we ran bwc tests.

Additionally, it creates 4 new projects that correspond to the different types of snapshots that may exist
for a given version. Its possible to now run those individual tasks to work out bwc logic whereas
previously it was impossible and the entire suite of bwc tests had to be run to work out any logic
changes in the build tools' bwc project. Please note that if the project does not make sense for the 
version that is current, that an error will be thrown from that individual project if an attempt is made to 
run it.

This should allow for automating the version bumps as well, since it removes all the hardcoded version
logic from the configs.

build.gradle

@@ -103,7 +103,7 @@ allprojects {
     isIdea = System.getProperty("idea.active") != null || gradle.startParameter.taskNames.contains('idea') || gradle.startParameter.taskNames.contains('cleanIdea')
 
     // for BWC testing
-    versionCollection = versions
+    bwcVersions = versions
 
     buildMetadata = buildMetadataMap
   }
@@ -122,13 +122,13 @@ task verifyVersions {
     Set<Version> knownVersions = new TreeSet<>(xml.versioning.versions.version.collect { it.text() }.findAll { it ==~ /\d\.\d\.\d/ }.collect { Version.fromString(it) })
 
     // Limit the known versions to those that should be index compatible, and are not future versions
-    knownVersions = knownVersions.findAll { it.major >= versions.currentVersion.major - 1 && it.before(VersionProperties.elasticsearch) }
+    knownVersions = knownVersions.findAll { it.major >= bwcVersions.currentVersion.major - 1 && it.before(VersionProperties.elasticsearch) }
 
     /* Limit the listed versions to those that have been marked as released.
      * Versions not marked as released don't get the same testing and we want
      * to make sure that we flip all unreleased versions to released as soon
      * as possible after release. */
-    Set<Version> actualVersions = new TreeSet<>(versions.versionsIndexCompatibleWithCurrent.findAll { false == it.snapshot })
+    Set<Version> actualVersions = new TreeSet<>(bwcVersions.indexCompatible.findAll { false == it.snapshot })
 
     // Finally, compare!
     if (knownVersions.equals(actualVersions) == false) {
@@ -219,13 +219,14 @@ subprojects {
     "org.elasticsearch.plugin:rank-eval-client:${version}": ':modules:rank-eval',
   ]
 
-  for (final Version version : versionCollection.versionsIndexCompatibleWithCurrent) {
-    if (version.branch != null) {
-      final String snapshotProject = ":distribution:bwc-snapshot-${version.branch}"
-      project(snapshotProject).ext.bwcVersion = version
-      ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${version}"] = snapshotProject
-      ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${version}"] = snapshotProject
-      ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${version}"] = snapshotProject
+  bwcVersions.snapshotProjectNames.each { snapshotName ->
+    Version snapshot = bwcVersions.getSnapshotForProject(snapshotName)
+    if (snapshot != null ) {
+      String snapshotProject = ":distribution:bwc:${snapshotName}"
+      project(snapshotProject).ext.bwcVersion = snapshot
+      ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${snapshot}"] = snapshotProject
+      ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${snapshot}"] = snapshotProject
+      ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${snapshot}"] = snapshotProject
     }
   }
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/Version.groovy

@@ -34,24 +34,29 @@ public class Version {
     final int revision
     final int id
     final boolean snapshot
-    final String branch
     /**
-     * Suffix on the version name. Unlike Version.java the build does not
-     * consider alphas and betas different versions, it just preserves the
-     * suffix that the version was declared with in Version.java.
+     * Suffix on the version name.
      */
     final String suffix
 
     public Version(int major, int minor, int revision,
-            String suffix, boolean snapshot, String branch) {
+            String suffix, boolean snapshot) {
         this.major = major
         this.minor = minor
         this.revision = revision
         this.snapshot = snapshot
         this.suffix = suffix
-        this.branch = branch
-        this.id = major * 100000 + minor * 1000 + revision * 10 +
-            (snapshot ? 1 : 0)
+
+        int suffixOffset = 0
+        if (suffix.contains("alpha")) {
+          suffixOffset += Integer.parseInt(suffix.substring(6))
+        } else if (suffix.contains("beta")) {
+          suffixOffset += 25 + Integer.parseInt(suffix.substring(5))
+        } else if (suffix.contains("rc")) {
+          suffixOffset += 50 + Integer.parseInt(suffix.substring(3));
+        }
+
+        this.id = major * 1000000 + minor * 10000 + revision * 100 + suffixOffset
     }
 
     public static Version fromString(String s) {
@@ -60,7 +65,7 @@ public class Version {
             throw new InvalidUserDataException("Invalid version [${s}]")
         }
         return new Version(m.group(1) as int, m.group(2) as int,
-            m.group(3) as int, m.group(4) ?: '', m.group(5) != null, null)
+            m.group(3) as int, m.group(4) ?: '', m.group(5) != null)
     }
 
     @Override

buildSrc/src/main/groovy/org/elasticsearch/gradle/VersionCollection.groovy

@@ -20,164 +20,336 @@
 package org.elasticsearch.gradle
 
 import org.gradle.api.GradleException
+import org.gradle.api.InvalidUserDataException
 
 import java.util.regex.Matcher
 
 /**
  * The collection of version constants declared in Version.java, for use in BWC testing.
+ *
+ *  if major+1 released: released artifacts from $version down to major-1.highestMinor.highestPatch, none of these should be snapshots, period.
+ *  if major+1 unreleased:
+ *  - if released:
+ *   -- caveat 0: snapshot for the major-1.highestMinor.highestPatch
+ *  - if unreleased:
+ *   -- caveat 0: snapshot for the major-1.highestMinor.highestPatch
+ *   -- caveat 1: every same major lower minor branch should also be tested if its released, and if not, its a snapshot. There should only be max 2 of these.
+ *   -- caveat 2: the largest released minor branch before the unreleased minor should also be a snapshot
+ *   -- caveat 3: if the current version is a different major than the previous rules apply to major - 1 of the current version
+ *
+ * Please note that the caveat's also correspond with the 4 types of snapshots.
+ * - Caveat 0 - always maintenanceBugfixSnapshot.
+ * - Caveat 1 - This is tricky. If caveat 3 applies, the highest matching value is nextMinorSnapshot, if there is another it is the stagedMinorSnapshot.
+ *              If caveat 3 does not apply then the only possible value is the stagedMinorSnapshot.
+ * - Caveat 2 - always nextBugfixSnapshot
+ * - Caveat 3 - this only changes the applicability of Caveat 1
+ *
+ * Notes on terminology:
+ * - The case for major+1 being released is accomplished through the isReleasableBranch value. If this is false, then the branch is no longer
+ *   releasable, meaning not to test against any snapshots.
+ * - Released is defined as having > 1 suffix-free version in a major.minor series. For instance, only 6.2.0 means unreleased, but a
+ *   6.2.0 and 6.2.1 mean that 6.2.0 was released already.
  */
 class VersionCollection {
 
     private final List<Version> versions
+    Version nextMinorSnapshot
+    Version stagedMinorSnapshot
+    Version nextBugfixSnapshot
+    Version maintenanceBugfixSnapshot
+    final Version currentVersion
+    private final TreeSet<Version> versionSet = new TreeSet<>()
+    final List<String> snapshotProjectNames = ['next-minor-snapshot',
+                                               'staged-minor-snapshot',
+                                               'next-bugfix-snapshot',
+                                               'maintenance-bugfix-snapshot']
+
+
+
+    // When we roll 8.0 its very likely these will need to be extracted from this class
     private final boolean buildSnapshot = System.getProperty("build.snapshot", "true") == "true"
+    private final boolean isReleasableBranch = true
 
     /**
-     * Construct a VersionCollection from the lines of the Version.java file.
+     * Construct a VersionCollection from the lines of the Version.java file. The basic logic for the following is pretty straight forward.
+
      * @param versionLines The lines of the Version.java file.
      */
     VersionCollection(List<String> versionLines) {
 
         List<Version> versions = []
+        // This class should be converted wholesale to use the treeset
 
         for (final String line : versionLines) {
             final Matcher match = line =~ /\W+public static final Version V_(\d+)_(\d+)_(\d+)(_alpha\d+|_beta\d+|_rc\d+)? .*/
             if (match.matches()) {
                 final Version foundVersion = new Version(
                         Integer.parseInt(match.group(1)), Integer.parseInt(match.group(2)),
-                        Integer.parseInt(match.group(3)), (match.group(4) ?: '').replace('_', '-'), false, null)
-
-                if (versions.size() > 0 && foundVersion.onOrBeforeIncludingSuffix(versions[-1])) {
-                    throw new GradleException("Versions.java contains out of order version constants:" +
-                            " ${foundVersion} should come before ${versions[-1]}")
-                }
-
-                // Only keep the last alpha/beta/rc in the series
-                if (versions.size() > 0 && versions[-1].id == foundVersion.id) {
-                    versions[-1] = foundVersion
-                } else {
-                    versions.add(foundVersion)
-                }
+                        Integer.parseInt(match.group(3)), (match.group(4) ?: '').replace('_', '-'), false)
+                safeAddToSet(foundVersion)
             }
         }
 
-        if (versions.empty) {
-            throw new GradleException("Unexpectedly found no version constants in Versions.java");
+        if (versionSet.empty) {
+            throw new GradleException("Unexpectedly found no version constants in Versions.java")
         }
 
-        /*
-         * The tip of each minor series (>= 5.6) is unreleased, so they must be built from source (we set branch to non-null), and we set
-         * the snapshot flag if and only if build.snapshot is true.
-         */
-        Version prevConsideredVersion = null
-        boolean found6xSnapshot = false
-        for (final int versionIndex = versions.size() - 1; versionIndex >= 0; versionIndex--) {
-            final Version currConsideredVersion = versions[versionIndex]
-
-            if (prevConsideredVersion == null
-                    || currConsideredVersion.major != prevConsideredVersion.major
-                    || currConsideredVersion.minor != prevConsideredVersion.minor) {
-
-                // This is a snapshot version. Work out its branch. NB this doesn't name the current branch correctly, but this doesn't
-                // matter as we don't BWC test against it.
-                String branch = "${currConsideredVersion.major}.${currConsideredVersion.minor}"
-
-                if (false == found6xSnapshot && currConsideredVersion.major == 6) {
-                    // TODO needs generalising to deal with when 7.x is cut, and when 6.x is deleted, and so on...
-                    branch = "6.x"
-                    found6xSnapshot = true
-                }
+        // If the major version has been released, then remove all of the alpha/beta/rc versions that exist in the set
+        versionSet.removeAll { it.suffix.isEmpty() == false && isMajorReleased(it, versionSet) }
 
-                versions[versionIndex] = new Version(
-                        currConsideredVersion.major, currConsideredVersion.minor,
-                        currConsideredVersion.revision, currConsideredVersion.suffix, buildSnapshot, branch)
-            }
+        // set currentVersion
+        Version lastVersion = versionSet.last()
+        currentVersion = new Version(lastVersion.major, lastVersion.minor, lastVersion.revision, lastVersion.suffix, buildSnapshot)
 
-            if (currConsideredVersion.onOrBefore("5.6.0")) {
-                break
-            }
+        // remove all of the potential alpha/beta/rc from the currentVersion
+        versionSet.removeAll {
+            it.suffix.isEmpty() == false &&
+            it.major == currentVersion.major &&
+            it.minor == currentVersion.minor &&
+            it.revision == currentVersion.revision }
+
+        // re-add the currentVersion to the set
+        versionSet.add(currentVersion)
 
-            prevConsideredVersion = currConsideredVersion
+        if (isReleasableBranch) {
+            if (isReleased(currentVersion)) {
+                // caveat 0 - if the minor has been released then it only has a maintenance version
+                // go back 1 version to get the last supported snapshot version of the line, which is a maint bugfix
+                Version highestMinor = getHighestPreviousMinor(currentVersion.major)
+                maintenanceBugfixSnapshot = replaceAsSnapshot(highestMinor)
+            } else {
+                // caveat 3 - if our currentVersion is a X.0.0, we need to check X-1 minors to see if they are released
+                if (currentVersion.minor == 0) {
+                    for (Version version: getMinorTips(currentVersion.major - 1)) {
+                        if (isReleased(version) == false) {
+                            // caveat 1 - This should only ever contain 2 non released branches in flight. An example is 6.x is frozen,
+                            // and 6.2 is cut but not yet released there is some simple logic to make sure that in the case of more than 2,
+                            // it will bail. The order is that the minor snapshot is fufilled first, and then the staged minor snapshot
+                            if (nextMinorSnapshot == null) {
+                                // it has not been set yet
+                                nextMinorSnapshot = replaceAsSnapshot(version)
+                            } else if (stagedMinorSnapshot == null) {
+                                stagedMinorSnapshot = replaceAsSnapshot(version)
+                            } else {
+                                throw new GradleException("More than 2 snapshot version existed for the next minor and staged (frozen) minors.")
+                            }
+                        } else {
+                            // caveat 2 - this is the last minor snap for this major, so replace the highest (last) one of these and break
+                            nextBugfixSnapshot = replaceAsSnapshot(version)
+                            // we only care about the largest minor here, so in the case of 6.1 and 6.0, it will only get 6.1
+                            break
+                        }
+                    }
+                    // caveat 0 - now dip back 2 versions to get the last supported snapshot version of the line
+                    Version highestMinor = getHighestPreviousMinor(currentVersion.major - 1)
+                    maintenanceBugfixSnapshot = replaceAsSnapshot(highestMinor)
+                } else {
+                    // caveat 3 did not apply. version is not a X.0.0, so we are somewhere on a X.Y line
+                    // only check till minor == 0 of the major
+                    for (Version version: getMinorTips(currentVersion.major)) {
+                        if (isReleased(version) == false) {
+                            // caveat 1 - This should only ever contain 0 or 1 branch in flight. An example is 6.x is frozen, and 6.2 is cut
+                            // but not yet released there is some simple logic to make sure that in the case of more than 1, it will bail
+                            if (stagedMinorSnapshot == null) {
+                                stagedMinorSnapshot = replaceAsSnapshot(version)
+                            } else {
+                                throw new GradleException("More than 1 snapshot version existed for the staged (frozen) minors.")
+                            }
+                        } else {
+                            // caveat 2 - this is the last minor snap for this major, so replace the highest (last) one of these and break
+                            nextBugfixSnapshot = replaceAsSnapshot(version)
+                            // we only care about the largest minor here, so in the case of 6.1 and 6.0, it will only get 6.1
+                            break
+                        }
+                    }
+                    // caveat 0 - now dip back 1 version to get the last supported snapshot version of the line
+                    Version highestMinor = getHighestPreviousMinor(currentVersion.major)
+                    maintenanceBugfixSnapshot = replaceAsSnapshot(highestMinor)
+                }
+            }
         }
 
-        this.versions = Collections.unmodifiableList(versions)
+        this.versions = Collections.unmodifiableList(versionSet.toList())
     }
 
     /**
      * @return The list of versions read from the Version.java file
      */
     List<Version> getVersions() {
-        return Collections.unmodifiableList(versions)
+        return versions
+    }
+
+    /**
+     * Index compat supports 1 previous entire major version. For instance, any 6.x test for this would test all of 5 up to that 6.x version
+     *
+     * @return All earlier versions that should be tested for index BWC with the current version.
+     */
+    List<Version> getIndexCompatible() {
+        int actualMajor = (currentVersion.major == 5 ? 2 : currentVersion.major - 1)
+        return versionSet
+            .tailSet(Version.fromString("${actualMajor}.0.0"))
+            .headSet(currentVersion)
+            .asList()
     }
 
     /**
-     * @return The latest version in the Version.java file, which must be the current version of the system.
+     * Ensures the types of snapshot are not null and are also in the index compat list
      */
-    Version getCurrentVersion() {
-        return versions[-1]
+    List<Version> getSnapshotsIndexCompatible() {
+        List<Version> compatSnapshots = []
+        List<Version> allCompatVersions = getIndexCompatible()
+        if (allCompatVersions.contains(nextMinorSnapshot)) {
+            compatSnapshots.add(nextMinorSnapshot)
+        }
+        if (allCompatVersions.contains(stagedMinorSnapshot)) {
+            compatSnapshots.add(stagedMinorSnapshot)
+        }
+        if (allCompatVersions.contains(nextBugfixSnapshot)) {
+            compatSnapshots.add(nextBugfixSnapshot)
+        }
+        if (allCompatVersions.contains(maintenanceBugfixSnapshot)) {
+            compatSnapshots.add(maintenanceBugfixSnapshot)
+        }
+
+        return compatSnapshots
     }
 
     /**
-     * @return The snapshot at the end of the previous minor series in the current major series, or null if this is the first minor series.
+     * Wire compat supports the last minor of the previous major. For instance, any 6.x test would test 5.6 up to that 6.x version
+     *
+     * @return All earlier versions that should be tested for wire BWC with the current version.
      */
-    Version getBWCSnapshotForCurrentMajor() {
-        return getLastSnapshotWithMajor(currentVersion.major)
+    List<Version> getWireCompatible() {
+        // Get the last minor of the previous major
+        Version lowerBound = getHighestPreviousMinor(currentVersion.major)
+        return versionSet
+            .tailSet(Version.fromString("${lowerBound.major}.${lowerBound.minor}.0"))
+            .headSet(currentVersion)
+            .toList()
     }
 
     /**
-     * @return The snapshot at the end of the previous major series, which must not be null.
+     * Ensures the types of snapshot are not null and are also in the wire compat list
      */
-    Version getBWCSnapshotForPreviousMajor() {
-        Version version = getLastSnapshotWithMajor(currentVersion.major - 1)
-        assert version != null : "getBWCSnapshotForPreviousMajor(): found no versions in the previous major"
-        return version
+    List<Version> getSnapshotsWireCompatible() {
+        List<Version> compatSnapshots = []
+        List<Version> allCompatVersions = getWireCompatible()
+        if (allCompatVersions.contains(nextMinorSnapshot)) {
+            compatSnapshots.add(nextMinorSnapshot)
+        }
+        if (allCompatVersions.contains(stagedMinorSnapshot)) {
+            compatSnapshots.add(stagedMinorSnapshot)
+        }
+        if (allCompatVersions.contains(nextBugfixSnapshot)) {
+            compatSnapshots.add(nextBugfixSnapshot)
+        }
+        if (allCompatVersions.contains(maintenanceBugfixSnapshot)) {
+            compatSnapshots.add(maintenanceBugfixSnapshot)
+        }
+        // There was no wire compat for the 2.x line
+        compatSnapshots.removeAll {it.major == 2}
+
+        return compatSnapshots
     }
 
-    private Version getLastSnapshotWithMajor(int targetMajor) {
-        final String currentVersion = currentVersion.toString()
-        final int snapshotIndex = versions.findLastIndexOf {
-            it.major == targetMajor && it.before(currentVersion) && it.snapshot == buildSnapshot
+    /**
+     * Grabs the proper snapshot based on the name passed in. These names should correspond with gradle project names under bwc. If you
+     * are editing this if/else it is only because you added another project under :distribution:bwc. Do not modify this method or its
+     * reasoning for throwing the exception unless you are sure that it will not harm :distribution:bwc.
+     */
+    Version getSnapshotForProject(String snapshotProjectName) {
+        if (snapshotProjectName == 'next-minor-snapshot') {
+            return nextMinorSnapshot
+        } else if (snapshotProjectName == 'staged-minor-snapshot') {
+            return stagedMinorSnapshot
+        } else if (snapshotProjectName == 'maintenance-bugfix-snapshot') {
+            return maintenanceBugfixSnapshot
+        } else if (snapshotProjectName == 'next-bugfix-snapshot') {
+            return nextBugfixSnapshot
+        } else {
+            throw new InvalidUserDataException("Unsupported project name ${snapshotProjectName}")
         }
-        return snapshotIndex == -1 ? null : versions[snapshotIndex]
     }
 
-    private List<Version> versionsOnOrAfterExceptCurrent(Version minVersion) {
-        final String minVersionString = minVersion.toString()
-        return Collections.unmodifiableList(versions.findAll {
-            it.onOrAfter(minVersionString) && it != currentVersion
-        })
+    /**
+     * Uses basic logic about our releases to determine if this version has been previously released
+     */
+    private boolean isReleased(Version version) {
+        return version.revision > 0
     }
 
     /**
-     * @return All earlier versions that should be tested for index BWC with the current version.
+     * Validates that the count of non suffixed (alpha/beta/rc) versions in a given major to major+1 is greater than 1.
+     * This means that there is more than just a major.0.0 or major.0.0-alpha in a branch to signify it has been prevously released.
+     */
+    private boolean isMajorReleased(Version version, TreeSet<Version> items) {
+        return items
+            .tailSet(Version.fromString("${version.major}.0.0"))
+            .headSet(Version.fromString("${version.major + 1}.0.0"))
+            .count { it.suffix.isEmpty() }  // count only non suffix'd versions as actual versions that may be released
+            .intValue() > 1
+    }
+
+    /**
+     * Gets the largest version previous major version based on the nextMajorVersion passed in.
+     * If you have a list [5.0.2, 5.1.2, 6.0.1, 6.1.1] and pass in 6 for the nextMajorVersion, it will return you 5.1.2
      */
-    List<Version> getVersionsIndexCompatibleWithCurrent() {
-        final Version firstVersionOfCurrentMajor = versions.find { it.major >= currentVersion.major - 1 }
-        return versionsOnOrAfterExceptCurrent(firstVersionOfCurrentMajor)
+    private Version getHighestPreviousMinor(Integer nextMajorVersion) {
+        return versionSet.headSet(Version.fromString("${nextMajorVersion}.0.0")).last()
     }
 
-    private Version getMinimumWireCompatibilityVersion() {
-        final int firstIndexOfThisMajor = versions.findIndexOf { it.major == currentVersion.major }
-        if (firstIndexOfThisMajor == 0) {
-            return versions[0]
+    /**
+     * Helper function for turning a version into a snapshot version, removing and readding it to the tree
+     */
+    private Version replaceAsSnapshot(Version version) {
+        versionSet.remove(version)
+        Version snapshotVersion = new Version(version.major, version.minor, version.revision, version.suffix, buildSnapshot)
+        safeAddToSet(snapshotVersion)
+        return snapshotVersion
+    }
+
+    /**
+     * Safely adds a value to the treeset, or bails if the value already exists.
+     * @param version
+     */
+    private void safeAddToSet(Version version) {
+        if (versionSet.add(version) == false) {
+            throw new GradleException("Versions.java contains duplicate entries for ${version}")
         }
-        final Version lastVersionOfEarlierMajor = versions[firstIndexOfThisMajor - 1]
-        return versions.find { it.major == lastVersionOfEarlierMajor.major && it.minor == lastVersionOfEarlierMajor.minor }
     }
 
     /**
-     * @return All earlier versions that should be tested for wire BWC with the current version.
+     * Gets the entire set of major.minor.* given those parameters.
      */
-    List<Version> getVersionsWireCompatibleWithCurrent() {
-        return versionsOnOrAfterExceptCurrent(minimumWireCompatibilityVersion)
+    private SortedSet<Version> getMinorSetForMajor(Integer major, Integer minor) {
+        return versionSet
+            .tailSet(Version.fromString("${major}.${minor}.0"))
+            .headSet(Version.fromString("${major}.${minor + 1}.0"))
     }
 
     /**
-     * `gradle check` does not run all BWC tests. This defines which tests it does run.
-     * @return Versions to test for BWC during gradle check.
+     * Gets the entire set of major.* to the currentVersion
      */
-    List<Version> getBasicIntegrationTestVersions() {
-        // TODO these are the versions checked by `gradle check` for BWC tests. Their choice seems a litle arbitrary.
-        List<Version> result = [BWCSnapshotForPreviousMajor, BWCSnapshotForCurrentMajor]
-        return Collections.unmodifiableList(result.findAll { it != null })
+    private SortedSet<Version> getMajorSet(Integer major) {
+        return versionSet
+            .tailSet(Version.fromString("${major}.0.0"))
+            .headSet(currentVersion)
+    }
+
+    /**
+     * Gets the tip of each minor set and puts it in a list.
+     *
+     * examples:
+     *  [1.0.0, 1.1.0, 1.1.1, 1.2.0, 1.3.1] will return [1.0.0, 1.1.1, 1.2.0, 1.3.1]
+     *  [1.0.0, 1.0.1, 1.0.2, 1.0.3, 1.0.4] will return [1.0.4]
+     */
+    private List<Version> getMinorTips(Integer major) {
+        TreeSet<Version> majorSet = getMajorSet(major)
+        List<Version> minorList = new ArrayList<>()
+        for (int minor = majorSet.last().minor; minor >= 0; minor--) {
+            TreeSet<Version> minorSetInMajor = getMinorSetForMajor(major, minor)
+            minorList.add(minorSetInMajor.last())
+        }
+        return minorList
     }
 }

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy

@@ -108,7 +108,7 @@ class VagrantTestPlugin implements Plugin<Project> {
         if (upgradeFromVersion == null) {
             String firstPartOfSeed = project.rootProject.testSeed.tokenize(':').get(0)
             final long seed = Long.parseUnsignedLong(firstPartOfSeed, 16)
-            final def indexCompatVersions = project.versionCollection.versionsIndexCompatibleWithCurrent
+            final def indexCompatVersions = project.bwcVersions.indexCompatible
             upgradeFromVersion = indexCompatVersions[new Random(seed).nextInt(indexCompatVersions.size())]
         }
 

buildSrc/src/test/groovy/org/elasticsearch/gradle/VersionCollectionTest.groovy

@@ -0,0 +1,226 @@
+package org.elasticsearch.gradle
+
+class VersionCollectionTest extends GroovyTestCase {
+
+  String formatVersion(String version) {
+    return " public static final Version V_${version.replaceAll("\\.", "_")} "
+  }
+  def allVersions = [formatVersion('5.0.0'), formatVersion('5.0.0_alpha1'), formatVersion('5.0.0_alpha2'), formatVersion('5.0.0_beta1'),
+                     formatVersion('5.0.0_rc1'),formatVersion('5.0.0_rc2'),formatVersion('5.0.1'), formatVersion('5.0.2'),
+                     formatVersion('5.1.1'), formatVersion('5.1.2'), formatVersion('5.2.0'), formatVersion('5.2.1'), formatVersion('6.0.0'),
+                     formatVersion('6.0.1'), formatVersion('6.1.0'), formatVersion('6.1.1'), formatVersion('6.2.0'), formatVersion('6.3.0'),
+                     formatVersion('7.0.0_alpha1'), formatVersion('7.0.0_alpha2')]
+
+  /**
+   * This validates the logic of being on a unreleased major branch with a staged major-1.minor sibling. This case happens when a version is
+   * branched from Major-1.x At the time of this writing 6.2 is unreleased and 6.3 is the 6.x branch. This test simulates the behavior
+   * from 7.0 perspective, or master at the time of this writing.
+   */
+  void testAgainstMajorUnreleasedWithExistingStagedMinorRelease() {
+    VersionCollection vc = new VersionCollection(allVersions)
+    assertNotNull(vc)
+    assertEquals(vc.nextMinorSnapshot, Version.fromString("6.3.0-SNAPSHOT"))
+    assertEquals(vc.stagedMinorSnapshot, Version.fromString("6.2.0-SNAPSHOT"))
+    assertEquals(vc.nextBugfixSnapshot, Version.fromString("6.1.1-SNAPSHOT"))
+    assertEquals(vc.maintenanceBugfixSnapshot, Version.fromString("5.2.1-SNAPSHOT"))
+
+    vc.indexCompatible.containsAll(vc.versions)
+
+    // This should contain the same list sans the current version
+    List indexCompatList =  [Version.fromString("6.0.0"), Version.fromString("6.0.1"),
+                             Version.fromString("6.1.0"), Version.fromString("6.1.1-SNAPSHOT"),
+                             Version.fromString("6.2.0-SNAPSHOT"), Version.fromString("6.3.0-SNAPSHOT")]
+    assertTrue(indexCompatList.containsAll(vc.indexCompatible))
+    assertTrue(vc.indexCompatible.containsAll(indexCompatList))
+
+    List wireCompatList = [Version.fromString("6.3.0-SNAPSHOT")]
+    assertTrue(wireCompatList.containsAll(vc.wireCompatible))
+    assertTrue(vc.wireCompatible.containsAll(wireCompatList))
+
+    assertEquals(vc.snapshotsIndexCompatible.size(), 3)
+    assertTrue(vc.snapshotsIndexCompatible.contains(Version.fromString("6.3.0-SNAPSHOT")))
+    assertTrue(vc.snapshotsIndexCompatible.contains(Version.fromString("6.2.0-SNAPSHOT")))
+    assertTrue(vc.snapshotsIndexCompatible.contains(Version.fromString("6.1.1-SNAPSHOT")))
+
+    assertEquals(vc.snapshotsWireCompatible.size(), 1)
+    assertEquals(vc.snapshotsWireCompatible.first(), Version.fromString("6.3.0-SNAPSHOT"))
+  }
+
+  /**
+   * This validates the logic of being on a unreleased major branch without a staged major-1.minor sibling. This case happens once a staged,
+   * unreleased minor is released. At the time of this writing 6.2 is unreleased, so adding a 6.2.1 simulates a 6.2 release. This test
+   * simulates the behavior from 7.0 perspective, or master at the time of this writing.
+   */
+  void testAgainstMajorUnreleasedWithoutStagedMinorRelease() {
+    List localVersion = allVersions.clone()
+    localVersion.add(formatVersion('6.2.1')) // release 6.2
+
+    VersionCollection vc = new VersionCollection(localVersion)
+    assertNotNull(vc)
+    assertEquals(vc.nextMinorSnapshot, Version.fromString("6.3.0-SNAPSHOT"))
+    assertEquals(vc.stagedMinorSnapshot, null)
+    assertEquals(vc.nextBugfixSnapshot, Version.fromString("6.2.1-SNAPSHOT"))
+    assertEquals(vc.maintenanceBugfixSnapshot, Version.fromString("5.2.1-SNAPSHOT"))
+
+    vc.indexCompatible.containsAll(vc.versions)
+
+    // This should contain the same list sans the current version
+    List indexCompatList =  [Version.fromString("6.0.0"), Version.fromString("6.0.1"),
+                             Version.fromString("6.1.0"), Version.fromString("6.1.1"),
+                             Version.fromString("6.2.0"), Version.fromString("6.2.1-SNAPSHOT"),
+                             Version.fromString("6.3.0-SNAPSHOT")]
+    assertTrue(indexCompatList.containsAll(vc.indexCompatible))
+    assertTrue(vc.indexCompatible.containsAll(indexCompatList))
+
+    List wireCompatList = [Version.fromString("6.3.0-SNAPSHOT")]
+    assertTrue(wireCompatList.containsAll(vc.wireCompatible))
+    assertTrue(vc.wireCompatible.containsAll(wireCompatList))
+
+    assertEquals(vc.snapshotsIndexCompatible.size(), 2)
+    assertTrue(vc.snapshotsIndexCompatible.contains(Version.fromString("6.3.0-SNAPSHOT")))
+    assertTrue(vc.snapshotsIndexCompatible.contains(Version.fromString("6.2.1-SNAPSHOT")))
+
+    assertEquals(vc.snapshotsWireCompatible.size(), 1)
+    assertEquals(vc.snapshotsWireCompatible.first(), Version.fromString("6.3.0-SNAPSHOT"))
+  }
+
+  /**
+   * This validates the logic of being on a unreleased minor branch with a staged minor sibling. This case happens when a version is
+   * branched from Major.x At the time of this writing 6.2 is unreleased and 6.3 is the 6.x branch. This test simulates the behavior
+   * from 6.3 perspective.
+   */
+  void testAgainstMinorReleasedBranch() {
+    List localVersion = allVersions.clone()
+    localVersion.removeAll { it.toString().contains('7_0_0')} // remove all the 7.x so that the actual version is 6.3 (6.x)
+    VersionCollection vc = new VersionCollection(localVersion)
+    assertNotNull(vc)
+    assertEquals(vc.nextMinorSnapshot, null)
+    assertEquals(vc.stagedMinorSnapshot, Version.fromString("6.2.0-SNAPSHOT"))
+    assertEquals(vc.nextBugfixSnapshot, Version.fromString("6.1.1-SNAPSHOT"))
+    assertEquals(vc.maintenanceBugfixSnapshot, Version.fromString("5.2.1-SNAPSHOT"))
+
+    // This should contain the same list sans the current version
+    List indexCompatList = vc.versions.subList(0, vc.versions.size() - 1)
+    assertTrue(indexCompatList.containsAll(vc.indexCompatible))
+    assertTrue(vc.indexCompatible.containsAll(indexCompatList))
+
+    List wireCompatList = [Version.fromString("5.2.0"), Version.fromString("5.2.1-SNAPSHOT"), Version.fromString("6.0.0"),
+                           Version.fromString("6.0.1"), Version.fromString("6.1.0"), Version.fromString("6.1.1-SNAPSHOT"),
+                           Version.fromString("6.2.0-SNAPSHOT")]
+    assertTrue(wireCompatList.containsAll(vc.wireCompatible))
+    assertTrue(vc.wireCompatible.containsAll(wireCompatList))
+
+    assertEquals(vc.snapshotsIndexCompatible.size(), 3)
+    assertTrue(vc.snapshotsIndexCompatible.contains(Version.fromString("6.2.0-SNAPSHOT")))
+    assertTrue(vc.snapshotsIndexCompatible.contains(Version.fromString("6.1.1-SNAPSHOT")))
+    assertTrue(vc.snapshotsIndexCompatible.contains(Version.fromString("5.2.1-SNAPSHOT")))
+
+    assertEquals(vc.snapshotsWireCompatible.size(), 3)
+    assertTrue(vc.snapshotsWireCompatible.contains(Version.fromString("6.2.0-SNAPSHOT")))
+    assertTrue(vc.snapshotsWireCompatible.contains(Version.fromString("6.1.1-SNAPSHOT")))
+    assertTrue(vc.snapshotsWireCompatible.contains(Version.fromString("5.2.1-SNAPSHOT")))
+  }
+
+  /**
+   * This validates the logic of being on a unreleased minor branch without a staged minor sibling. This case happens once a staged,
+   * unreleased minor is released. At the time of this writing 6.2 is unreleased, so adding a 6.2.1 simulates a 6.2 release. This test
+   * simulates the behavior from 6.3 perspective.
+   */
+  void testAgainstMinorReleasedBranchNoStagedMinor() {
+    List localVersion = allVersions.clone()
+    // remove all the 7.x and add a 6.2.1 which means 6.2 was released
+    localVersion.removeAll { it.toString().contains('7_0_0')}
+    localVersion.add(formatVersion('6.2.1'))
+    VersionCollection vc = new VersionCollection(localVersion)
+    assertNotNull(vc)
+    assertEquals(vc.nextMinorSnapshot, null)
+    assertEquals(vc.stagedMinorSnapshot, null)
+    assertEquals(vc.nextBugfixSnapshot, Version.fromString("6.2.1-SNAPSHOT"))
+    assertEquals(vc.maintenanceBugfixSnapshot, Version.fromString("5.2.1-SNAPSHOT"))
+
+    // This should contain the same list sans the current version
+    List indexCompatList = vc.versions.subList(0, vc.versions.size() - 1)
+    assertTrue(indexCompatList.containsAll(vc.indexCompatible))
+    assertTrue(vc.indexCompatible.containsAll(indexCompatList))
+
+    List wireCompatList = [Version.fromString("5.2.0"), Version.fromString("5.2.1-SNAPSHOT"), Version.fromString("6.0.0"),
+                           Version.fromString("6.0.1"), Version.fromString("6.1.0"), Version.fromString("6.1.1"),
+                           Version.fromString("6.2.0"), Version.fromString("6.2.1-SNAPSHOT")]
+    assertTrue(wireCompatList.containsAll(vc.wireCompatible))
+    assertTrue(vc.wireCompatible.containsAll(wireCompatList))
+
+    assertEquals(vc.snapshotsIndexCompatible.size(), 2)
+    assertTrue(vc.snapshotsIndexCompatible.contains(Version.fromString("6.2.1-SNAPSHOT")))
+    assertTrue(vc.snapshotsIndexCompatible.contains(Version.fromString("5.2.1-SNAPSHOT")))
+
+    assertEquals(vc.snapshotsWireCompatible.size(), 2)
+    assertTrue(vc.snapshotsWireCompatible.contains(Version.fromString("6.2.1-SNAPSHOT")))
+    assertTrue(vc.snapshotsWireCompatible.contains(Version.fromString("5.2.1-SNAPSHOT")))
+  }
+
+  /**
+   * This validates the logic of being on a released minor branch. At the time of writing, 6.2 is unreleased, so this is equivalent of being
+   * on 6.1.
+   */
+  void testAgainstOldMinor() {
+
+    List localVersion = allVersions.clone()
+    // remove the 7 alphas and the ones greater than 6.1
+    localVersion.removeAll { it.toString().contains('7_0_0') || it.toString().contains('V_6_2') || it.toString().contains('V_6_3') }
+    VersionCollection vc = new VersionCollection(localVersion)
+    assertNotNull(vc)
+    assertEquals(vc.nextMinorSnapshot, null)
+    assertEquals(vc.stagedMinorSnapshot, null)
+    assertEquals(vc.nextBugfixSnapshot, null)
+    assertEquals(vc.maintenanceBugfixSnapshot, Version.fromString("5.2.1-SNAPSHOT"))
+
+    // This should contain the same list sans the current version
+    List indexCompatList = vc.versions.subList(0, vc.versions.size() - 1)
+    assertTrue(indexCompatList.containsAll(vc.indexCompatible))
+    assertTrue(vc.indexCompatible.containsAll(indexCompatList))
+
+    List wireCompatList = [Version.fromString("5.2.0"), Version.fromString("5.2.1-SNAPSHOT"), Version.fromString("6.0.0"),
+                           Version.fromString("6.0.1"), Version.fromString("6.1.0")]
+    assertTrue(wireCompatList.containsAll(vc.wireCompatible))
+    assertTrue(vc.wireCompatible.containsAll(wireCompatList))
+
+    assertEquals(vc.snapshotsIndexCompatible.size(), 1)
+    assertTrue(vc.snapshotsIndexCompatible.contains(Version.fromString("5.2.1-SNAPSHOT")))
+
+    assertEquals(vc.snapshotsWireCompatible.size(), 1)
+    assertTrue(vc.snapshotsWireCompatible.contains(Version.fromString("5.2.1-SNAPSHOT")))
+  }
+
+  /**
+   * This validates the lower bound of wire compat, which is 5.0. It also validates that the span of 2.x to 5.x if it is decided to port
+   * this fix all the way to the maint 5.6 release.
+   */
+  void testFloorOfWireCompatVersions() {
+    List localVersion = [formatVersion('2.0.0'), formatVersion('2.0.1'), formatVersion('2.1.0'), formatVersion('2.1.1'),
+                          formatVersion('5.0.0'), formatVersion('5.0.1'), formatVersion('5.1.0'), formatVersion('5.1.1'),
+                          formatVersion('5.2.0'),formatVersion('5.2.1'),formatVersion('5.3.0'),formatVersion('5.3.1'),
+                          formatVersion('5.3.2')]
+    VersionCollection vc = new VersionCollection(localVersion)
+    assertNotNull(vc)
+    assertEquals(vc.maintenanceBugfixSnapshot, Version.fromString("2.1.1-SNAPSHOT"))
+
+    // This should contain the same list sans the current version
+    List indexCompatList = vc.versions.subList(0, vc.versions.size() - 1)
+    assertTrue(indexCompatList.containsAll(vc.indexCompatible))
+    assertTrue(vc.indexCompatible.containsAll(indexCompatList))
+
+    List wireCompatList = [Version.fromString("2.1.0"), Version.fromString("2.1.1-SNAPSHOT"), Version.fromString("5.0.0"),
+                           Version.fromString("5.0.1"), Version.fromString("5.1.0"),
+                           Version.fromString("5.1.1"), Version.fromString("5.2.0"), Version.fromString("5.2.1"),
+                           Version.fromString("5.3.0"), Version.fromString("5.3.1")]
+
+    assertTrue(wireCompatList.containsAll(vc.wireCompatible))
+    assertTrue(vc.wireCompatible.containsAll(wireCompatList))
+
+    assertEquals(vc.snapshotsIndexCompatible.size(), 1)
+    assertTrue(vc.snapshotsIndexCompatible.contains(Version.fromString("2.1.1-SNAPSHOT")))
+
+    // ensure none of the 2.x snapshots appear here, as this is the floor of bwc for wire compat
+    assertEquals(vc.snapshotsWireCompatible.size(), 0)
+  }
+}

distribution/bwc/build.gradle

@@ -29,14 +29,21 @@ import java.util.regex.Matcher
  * tests to test against the next unreleased version, closest to this version,
  * without relying on snapshots.
  */
-final Matcher match = project.name =~ /bwc-snapshot-(\d+\.(\d+|x))/
-if (!match.matches()) {
-  throw new InvalidUserDataException("Unsupport project name ${project.name}")
-}
-String bwcBranch = match.group(1)
+subprojects {
+
+  Version bwcVersion = bwcVersions.getSnapshotForProject(project.name)
+  if (bwcVersion == null) {
+    // this project wont do anything
+    return
+  }
 
-if (project.hasProperty('bwcVersion')) {
-  Version bwcVersion = project.ext.bwcVersion
+  String bwcBranch
+  if (project.name == 'next-minor-snapshot') {
+    // this is always a .x series
+    bwcBranch = "${bwcVersion.major}.x"
+  } else {
+    bwcBranch = "${bwcVersion.major}.${bwcVersion.minor}"
+  }
 
   apply plugin: 'distribution'
   // Not published so no need to assemble

qa/full-cluster-restart/build.gradle

@@ -19,6 +19,7 @@
 
 
 import org.elasticsearch.gradle.Version
+import org.elasticsearch.gradle.VersionCollection
 import org.elasticsearch.gradle.test.RestIntegTestTask
 
 apply plugin: 'elasticsearch.standalone-test'
@@ -30,7 +31,7 @@ task bwcTest {
   group = 'verification'
 }
 
-for (Version version : versionCollection.versionsIndexCompatibleWithCurrent) {
+for (Version version : bwcVersions.indexCompatible) {
   String baseName = "v${version}"
 
   Task oldClusterTest = tasks.create(name: "${baseName}#oldClusterTest", type: RestIntegTestTask) {
@@ -105,10 +106,11 @@ test.enabled = false // no unit tests for rolling upgrades, only the rest integr
 // basic integ tests includes testing bwc against the most recent version
 task integTest {
   if (project.bwc_tests_enabled) {
-    for (final def version : versionCollection.basicIntegrationTestVersions) {
+    for (final def version : bwcVersions.snapshotsIndexCompatible) {
       dependsOn "v${version}#bwcTest"
     }
   }
 }
 
 check.dependsOn(integTest)
+

qa/mixed-cluster/build.gradle

@@ -29,7 +29,7 @@ task bwcTest {
   group = 'verification'
 }
 
-for (Version version : versionCollection.versionsWireCompatibleWithCurrent) {
+for (Version version : bwcVersions.wireCompatible) {
   String baseName = "v${version}"
 
   Task mixedClusterTest = tasks.create(name: "${baseName}#mixedClusterTest", type: RestIntegTestTask) {
@@ -66,7 +66,7 @@ test.enabled = false // no unit tests for rolling upgrades, only the rest integr
 // basic integ tests includes testing bwc against the most recent version
 task integTest {
   if (project.bwc_tests_enabled) {
-    for (final def version : versionCollection.basicIntegrationTestVersions) {
+    for (final def version : bwcVersions.snapshotsWireCompatible) {
       dependsOn "v${version}#bwcTest"
     }
   }

qa/query-builder-bwc/build.gradle

@@ -30,7 +30,7 @@ task bwcTest {
     group = 'verification'
 }
 
-for (Version version : versionCollection.versionsIndexCompatibleWithCurrent) {
+for (Version version : bwcVersions.indexCompatible) {
     String baseName = "v${version}"
 
     Task oldQueryBuilderTest = tasks.create(name: "${baseName}#oldQueryBuilderTest", type: RestIntegTestTask) {
@@ -82,11 +82,10 @@ test.enabled = false // no unit tests for rolling upgrades, only the rest integr
 
 // basic integ tests includes testing bwc against the most recent version
 task integTest {
-    if (project.bwc_tests_enabled) {
-        for (final def version : versionCollection.basicIntegrationTestVersions) {
-            dependsOn "v${version}#bwcTest"
-        }
-    }
+  if (project.bwc_tests_enabled) {
+      final def version = bwcVersions.snapshotsIndexCompatible.first()
+      dependsOn "v${version}#bwcTest"
+  }
 }
 
 check.dependsOn(integTest)

qa/rolling-upgrade/build.gradle

@@ -29,7 +29,7 @@ task bwcTest {
   group = 'verification'
 }
 
-for (Version version : versionCollection.versionsWireCompatibleWithCurrent) {
+for (Version version : bwcVersions.wireCompatible) {
   String baseName = "v${version}"
 
   Task oldClusterTest = tasks.create(name: "${baseName}#oldClusterTest", type: RestIntegTestTask) {
@@ -110,7 +110,7 @@ test.enabled = false // no unit tests for rolling upgrades, only the rest integr
 // basic integ tests includes testing bwc against the most recent version
 task integTest {
   if (project.bwc_tests_enabled) {
-    for (final def version : versionCollection.basicIntegrationTestVersions) {
+    for (final def version : bwcVersions.snapshotsWireCompatible) {
       dependsOn "v${version}#bwcTest"
     }
   }

qa/verify-version-constants/build.gradle

@@ -31,7 +31,7 @@ task bwcTest {
     group = 'verification'
 }
 
-for (Version version : versionCollection.versionsIndexCompatibleWithCurrent) {
+for (Version version : bwcVersions.indexCompatible) {
     String baseName = "v${version}"
     Task oldClusterTest = tasks.create(name: "${baseName}#oldClusterTest", type: RestIntegTestTask) {
         mustRunAfter(precommit)
@@ -57,9 +57,10 @@ for (Version version : versionCollection.versionsIndexCompatibleWithCurrent) {
 test.enabled = false
 
 task integTest {
-    for (final def version : versionCollection.basicIntegrationTestVersions) {
-        dependsOn "v${version}#bwcTest"
-    }
+  if (project.bwc_tests_enabled) {
+    final def version = bwcVersions.snapshotsIndexCompatible.first()
+    dependsOn "v${version}#bwcTest"
+  }
 }
 
 task verifyDocsLuceneVersion {

server/src/main/java/org/elasticsearch/Version.java

@@ -137,9 +137,6 @@ public class Version implements Comparable<Version> {
     public static final int V_6_0_1_ID = 6000199;
     public static final Version V_6_0_1 =
         new Version(V_6_0_1_ID, org.apache.lucene.util.Version.LUCENE_7_0_1);
-    public static final int V_6_0_2_ID = 6000299;
-    public static final Version V_6_0_2 =
-        new Version(V_6_0_2_ID, org.apache.lucene.util.Version.LUCENE_7_0_1);
     public static final int V_6_1_0_ID = 6010099;
     public static final Version V_6_1_0 = new Version(V_6_1_0_ID, org.apache.lucene.util.Version.LUCENE_7_1_0);
     public static final int V_6_1_1_ID = 6010199;
@@ -148,8 +145,6 @@ public class Version implements Comparable<Version> {
     public static final Version V_6_1_2 = new Version(V_6_1_2_ID, org.apache.lucene.util.Version.LUCENE_7_1_0);
     public static final int V_6_1_3_ID = 6010399;
     public static final Version V_6_1_3 = new Version(V_6_1_3_ID, org.apache.lucene.util.Version.LUCENE_7_1_0);
-    public static final int V_6_1_4_ID = 6010499;
-    public static final Version V_6_1_4 = new Version(V_6_1_4_ID, org.apache.lucene.util.Version.LUCENE_7_1_0);
     public static final int V_6_2_0_ID = 6020099;
     public static final Version V_6_2_0 = new Version(V_6_2_0_ID, org.apache.lucene.util.Version.LUCENE_7_2_1);
     public static final int V_6_2_1_ID = 6020199;
@@ -184,8 +179,6 @@ public class Version implements Comparable<Version> {
                 return V_6_2_1;
             case V_6_2_0_ID:
                 return V_6_2_0;
-            case V_6_1_4_ID:
-                return V_6_1_4;
             case V_6_1_3_ID:
                 return V_6_1_3;
             case V_6_1_2_ID:
@@ -194,8 +187,6 @@ public class Version implements Comparable<Version> {
                 return V_6_1_1;
             case V_6_1_0_ID:
                 return V_6_1_0;
-            case V_6_0_2_ID:
-                return V_6_0_2;
             case V_6_0_1_ID:
                 return V_6_0_1;
             case V_6_0_0_ID:

settings.gradle

@@ -1,3 +1,5 @@
+import org.elasticsearch.gradle.Version
+
 String dirName = rootProject.projectDir.name
 rootProject.name = dirName
 
@@ -18,6 +20,10 @@ List projects = [
   'distribution:tar',
   'distribution:deb',
   'distribution:rpm',
+  'distribution:bwc:next-minor-snapshot',
+  'distribution:bwc:staged-minor-snapshot',
+  'distribution:bwc:next-bugfix-snapshot',
+  'distribution:bwc:maintenance-bugfix-snapshot',
   'distribution:tools:launchers',
   'distribution:tools:plugin-cli',
   'server',
@@ -36,7 +42,7 @@ List projects = [
   * of the dir hierarchy to have a build.gradle. Otherwise we would have to iterate
   * all files/directories in the source tree to find all projects.
   */
-void addSubProjects(String path, File dir, List<String> projects, List<String> branches) {
+void addSubProjects(String path, File dir) {
   if (dir.isDirectory() == false) return;
   if (dir.name == 'buildSrc') return;
   if (new File(dir, 'build.gradle').exists() == false) return;
@@ -44,44 +50,28 @@ void addSubProjects(String path, File dir, List<String> projects, List<String> b
 
   final String projectName = "${path}:${dir.name}"
   include projectName
-
-  if (dir.name == 'bwc-snapshot-dummy-projects') {
-    for (final String branch : branches) {
-      final String snapshotProjectName = "${projectName}:bwc-snapshot-${branch}"
-      projects.add(snapshotProjectName)
-      include snapshotProjectName
-      project("${snapshotProjectName}").projectDir = dir
-    }
-    // TODO do we want to assert that there's nothing else in the bwc directory?
-  } else {
-    if (path.isEmpty() || path.startsWith(':example-plugins')) {
+  if (path.isEmpty() || path.startsWith(':example-plugins')) {
       project(projectName).projectDir = dir
     }
     for (File subdir : dir.listFiles()) {
-      addSubProjects(projectName, subdir, projects, branches)
+      addSubProjects(projectName, subdir)
     }
   }
-}
+
 
 // include example plugins first, so adding plugin dirs below won't muck with :example-plugins
 File examplePluginsDir = new File(rootProject.projectDir, 'plugins/examples')
 for (File example : examplePluginsDir.listFiles()) {
   if (example.isDirectory() == false) continue;
   if (example.name.startsWith('build') || example.name.startsWith('.')) continue;
-  addSubProjects(':example-plugins', example, projects, [])
+  addSubProjects(':example-plugins', example)
 }
 project(':example-plugins').projectDir = new File(rootProject.projectDir, 'plugins/examples')
 
-addSubProjects('', new File(rootProject.projectDir, 'libs'), projects, [])
-addSubProjects('', new File(rootProject.projectDir, 'modules'), projects, [])
-addSubProjects('', new File(rootProject.projectDir, 'plugins'), projects, [])
-addSubProjects('', new File(rootProject.projectDir, 'qa'), projects, [])
-
-/* Create projects for building BWC snapshot distributions from the heads of other branches */
-final List<String> branches = ['5.6', '6.0', '6.1', '6.2', '6.x']
-for (final String branch : branches) {
-  projects.add("distribution:bwc-snapshot-${branch}".toString())
-}
+addSubProjects('', new File(rootProject.projectDir, 'libs'))
+addSubProjects('', new File(rootProject.projectDir, 'modules'))
+addSubProjects('', new File(rootProject.projectDir, 'plugins'))
+addSubProjects('', new File(rootProject.projectDir, 'qa'))
 
 boolean isEclipse = System.getProperty("eclipse.launcher") != null || gradle.startParameter.taskNames.contains('eclipse') || gradle.startParameter.taskNames.contains('cleanEclipse')
 if (isEclipse) {
@@ -97,12 +87,6 @@ include projects.toArray(new String[0])
 
 project(':build-tools').projectDir = new File(rootProject.projectDir, 'buildSrc')
 
-/* The BWC snapshot projects share the same build directory and build file,
- * but apply to different backwards compatibility branches. */
-for (final String branch : branches) {
-  project(":distribution:bwc-snapshot-${branch}").projectDir = new File(rootProject.projectDir, 'distribution/bwc')
-}
-
 if (isEclipse) {
   project(":server").projectDir = new File(rootProject.projectDir, 'server/src/main')
   project(":server").buildFileName = 'eclipse-build.gradle'
@@ -126,7 +110,6 @@ if (isEclipse) {
 File extraProjects = new File(rootProject.projectDir.parentFile, "${dirName}-extra")
 if (extraProjects.exists()) {
   for (File extraProjectDir : extraProjects.listFiles()) {
-    addSubProjects('', extraProjectDir, projects, branches)
+    addSubProjects('', extraProjectDir)
   }
-}
-
+}

test/framework/build.gradle

@@ -71,6 +71,6 @@ task namingConventionsMain(type: org.elasticsearch.gradle.precommit.NamingConven
 precommit.dependsOn namingConventionsMain
 
 test.configure {
-  systemProperty 'tests.gradle_index_compat_versions', versionCollection.versionsIndexCompatibleWithCurrent.join(',')
-  systemProperty 'tests.gradle_wire_compat_versions', versionCollection.versionsWireCompatibleWithCurrent.join(',')
+  systemProperty 'tests.gradle_index_compat_versions', bwcVersions.indexCompatible.join(',')
+  systemProperty 'tests.gradle_wire_compat_versions', bwcVersions.wireCompatible.join(',')
 }

test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java

@@ -24,7 +24,10 @@ import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.collect.Tuple;
 
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
@@ -55,61 +58,59 @@ public class VersionUtils {
 
         Version last = versions.remove(versions.size() - 1);
         assert last.equals(current) : "The highest version must be the current one "
-            + "but was [" + last + "] and current was [" + current + "]";
-
-        /* In the 5.x series prior to 5.6, unreleased version constants had an
-         * `_UNRELEASED` suffix, and when making the first release on a minor release
-         * branch the last, unreleased, version constant from the previous minor branch
-         * was dropped. After 5.6, there is no `_UNRELEASED` suffix on version constants'
-         * names and, additionally, they are not dropped when a new minor release branch
-         * starts.
-         *
-         * This means that in 6.x and later series the last release _in each
-         * minor branch_ is unreleased, whereas in 5.x it's more complicated: There were
-         * (sometimes, and sometimes multiple) minor branches containing no releases, each
-         * of which contains a single version constant of the form 5.n.0, and these
-         * branches always followed a branch that _did_ contain a version of the
-         * form 5.m.p (p>0). All versions strictly before the last 5.m version are released,
-         * and all other 5.* versions are unreleased.
-         */
-
-        if (current.major == 5 && current.revision != 0) {
-            /* The current (i.e. latest) version is 5.a.b, b nonzero, which
-             * means that all other versions are released. */
+            + "but was [" + versions.get(versions.size() - 1) + "] and current was [" + current + "]";
+
+        if (current.revision != 0) {
+            /* If we are in a stable branch there should be no unreleased version constants
+             * because we don't expect to release any new versions in older branches. If there
+             * are extra constants then gradle will yell about it. */
             return new Tuple<>(unmodifiableList(versions), singletonList(current));
         }
 
-        final List<Version> unreleased = new ArrayList<>();
-        unreleased.add(current);
-        Version prevConsideredVersion = current;
-
-        for (int i = versions.size() - 1; i >= 0; i--) {
-            Version currConsideredVersion = versions.get(i);
-            if (currConsideredVersion.major == 5) {
-                unreleased.add(currConsideredVersion);
-                versions.remove(i);
-                if (currConsideredVersion.revision != 0) {
-                    /* Currently considering the latest version in the 5.x series,
-                     * which is (a) unreleased and (b) the only such. So we're done. */
-                    break;
-                }
-                /* ... else we're on a version of the form 5.n.0, and have not yet
-                 * considered a version of the form 5.n.m (m>0), so this entire branch
-                 * is unreleased, so carry on looking for a branch containing releases.
-                 */
-            } else if (currConsideredVersion.major != prevConsideredVersion.major
-                || currConsideredVersion.minor != prevConsideredVersion.minor) {
-                /* Have moved to the end of a new minor branch, so this is
-                 * an unreleased version. */
-                unreleased.add(currConsideredVersion);
-                versions.remove(i);
+        /* If we are on a patch release then we know that at least the version before the
+         * current one is unreleased. If it is released then gradle would be complaining. */
+        int unreleasedIndex = versions.size() - 1;
+        while (true) {
+            if (unreleasedIndex < 0) {
+                throw new IllegalArgumentException("Couldn't find first non-alpha release");
             }
-            prevConsideredVersion = currConsideredVersion;
-
+            /* We don't support backwards compatibility for alphas, betas, and rcs. But
+             * they were released so we add them to the released list. Usually this doesn't
+             * matter to consumers, but consumers that do care should filter non-release
+             * versions. */
+            if (versions.get(unreleasedIndex).isRelease()) {
+                break;
+            }
+            unreleasedIndex--;
         }
 
-        Collections.reverse(unreleased);
-        return new Tuple<>(unmodifiableList(versions), unmodifiableList(unreleased));
+        Version unreleased = versions.remove(unreleasedIndex);
+        if (unreleased.revision == 0) {
+            /*
+             * If the last unreleased version is itself a patch release then Gradle enforces that there is yet another unreleased version
+             * before that. However, we have to skip alpha/betas/RCs too (e.g., consider when the version constants are ..., 5.6.3, 5.6.4,
+             * 6.0.0-alpha1, ..., 6.0.0-rc1, 6.0.0-rc2, 6.0.0, 6.1.0 on the 6.x branch. In this case, we will have pruned 6.0.0 and 6.1.0 as
+             * unreleased versions, but we also need to prune 5.6.4. At this point though, unreleasedIndex will be pointing to 6.0.0-rc2, so
+             * we have to skip backwards until we find a non-alpha/beta/RC again. Then we can prune that version as an unreleased version
+             * too.
+             */
+            do {
+                unreleasedIndex--;
+            } while (versions.get(unreleasedIndex).isRelease() == false);
+            Version earlierUnreleased = versions.remove(unreleasedIndex);
+
+            // This earlierUnreleased is either the snapshot on the minor branch lower, or its possible its a staged release. If it is a
+            // staged release, remove it and return it in unreleased as well.
+            if (earlierUnreleased.revision == 0) {
+                unreleasedIndex--;
+                Version actualUnreleasedPreviousMinor = versions.remove(unreleasedIndex);
+                return new Tuple<>(unmodifiableList(versions), unmodifiableList(Arrays.asList(actualUnreleasedPreviousMinor,
+                    earlierUnreleased, unreleased, current)));
+            }
+
+            return new Tuple<>(unmodifiableList(versions), unmodifiableList(Arrays.asList(earlierUnreleased, unreleased, current)));
+        }
+        return new Tuple<>(unmodifiableList(versions), unmodifiableList(Arrays.asList(unreleased, current)));
     }
 
     private static final List<Version> RELEASED_VERSIONS;

test/framework/src/test/java/org/elasticsearch/test/VersionUtilsTests.java

@@ -269,25 +269,26 @@ public class VersionUtilsTests extends ESTestCase {
         final List<Version> expectedUnreleased;
         if (Booleans.parseBoolean(System.getProperty("build.snapshot", "true"))) {
             expectedReleased = Arrays.asList(
-                    TestNewMajorRelease.V_5_6_0,
-                    TestNewMajorRelease.V_5_6_1,
-                    TestNewMajorRelease.V_6_0_0_alpha1,
-                    TestNewMajorRelease.V_6_0_0_alpha2,
-                    TestNewMajorRelease.V_6_0_0_beta1,
-                    TestNewMajorRelease.V_6_0_0_beta2,
-                    TestNewMajorRelease.V_6_0_0);
-            expectedUnreleased = Arrays.asList(TestNewMajorRelease.V_5_6_2, TestNewMajorRelease.V_6_0_1);
+                TestNewMajorRelease.V_5_6_0,
+                TestNewMajorRelease.V_5_6_1,
+                TestNewMajorRelease.V_5_6_2,
+                TestNewMajorRelease.V_6_0_0_alpha1,
+                TestNewMajorRelease.V_6_0_0_alpha2,
+                TestNewMajorRelease.V_6_0_0_beta1,
+                TestNewMajorRelease.V_6_0_0_beta2,
+                TestNewMajorRelease.V_6_0_0);
+            expectedUnreleased = Arrays.asList(TestNewMajorRelease.V_6_0_1);
         } else {
             expectedReleased = Arrays.asList(
-                    TestNewMajorRelease.V_5_6_0,
-                    TestNewMajorRelease.V_5_6_1,
-                    TestNewMajorRelease.V_5_6_2,
-                    TestNewMajorRelease.V_6_0_0_alpha1,
-                    TestNewMajorRelease.V_6_0_0_alpha2,
-                    TestNewMajorRelease.V_6_0_0_beta1,
-                    TestNewMajorRelease.V_6_0_0_beta2,
-                    TestNewMajorRelease.V_6_0_0,
-                    TestNewMajorRelease.V_6_0_1);
+                TestNewMajorRelease.V_5_6_0,
+                TestNewMajorRelease.V_5_6_1,
+                TestNewMajorRelease.V_5_6_2,
+                TestNewMajorRelease.V_6_0_0_alpha1,
+                TestNewMajorRelease.V_6_0_0_alpha2,
+                TestNewMajorRelease.V_6_0_0_beta1,
+                TestNewMajorRelease.V_6_0_0_beta2,
+                TestNewMajorRelease.V_6_0_0,
+                TestNewMajorRelease.V_6_0_1);
             expectedUnreleased = Collections.emptyList();
         }
 
@@ -322,12 +323,13 @@ public class VersionUtilsTests extends ESTestCase {
             expectedReleased = Arrays.asList(
                     TestVersionBumpIn6x.V_5_6_0,
                     TestVersionBumpIn6x.V_5_6_1,
+                    TestVersionBumpIn6x.V_5_6_2,
                     TestVersionBumpIn6x.V_6_0_0_alpha1,
                     TestVersionBumpIn6x.V_6_0_0_alpha2,
                     TestVersionBumpIn6x.V_6_0_0_beta1,
                     TestVersionBumpIn6x.V_6_0_0_beta2,
                     TestVersionBumpIn6x.V_6_0_0);
-            expectedUnreleased = Arrays.asList(TestVersionBumpIn6x.V_5_6_2, TestVersionBumpIn6x.V_6_0_1, TestVersionBumpIn6x.V_6_1_0);
+            expectedUnreleased = Arrays.asList(TestVersionBumpIn6x.V_6_0_1, TestVersionBumpIn6x.V_6_1_0);
         } else {
             expectedReleased = Arrays.asList(
                     TestVersionBumpIn6x.V_5_6_0,
@@ -376,16 +378,16 @@ public class VersionUtilsTests extends ESTestCase {
             expectedReleased = Arrays.asList(
                     TestNewMinorBranchIn6x.V_5_6_0,
                     TestNewMinorBranchIn6x.V_5_6_1,
+                    TestNewMinorBranchIn6x.V_5_6_2,
                     TestNewMinorBranchIn6x.V_6_0_0_alpha1,
                     TestNewMinorBranchIn6x.V_6_0_0_alpha2,
                     TestNewMinorBranchIn6x.V_6_0_0_beta1,
                     TestNewMinorBranchIn6x.V_6_0_0_beta2,
                     TestNewMinorBranchIn6x.V_6_0_0,
+                    TestNewMinorBranchIn6x.V_6_0_1,
                     TestNewMinorBranchIn6x.V_6_1_0,
                     TestNewMinorBranchIn6x.V_6_1_1);
             expectedUnreleased = Arrays.asList(
-                    TestNewMinorBranchIn6x.V_5_6_2,
-                    TestNewMinorBranchIn6x.V_6_0_1,
                     TestNewMinorBranchIn6x.V_6_1_2,
                     TestNewMinorBranchIn6x.V_6_2_0);
         } else {