Add Vagrant Gradle plugin

This commit changes the current :elactisearch:qa:vagrant build file and transforms it into a Gradle plugin in order to reuse it in other projects.

Most of the code from the build.gradle file has been moved into the VagrantTestPlugin class. To avoid duplicated VMs when running vagrant tests, the Gradle plugin sets the following environment variables before running vagrant commands:
VAGRANT_CWD: absolute path to the folder that contains the Vagrantfile
VAGRANT_PROJECT_DIR: absolute path to the Gradle project that use the VagrantTestPlugin

The VAGRANT_PROJECT_DIR is used to share project folders and files with the vagrant VM. These folders and files are exported when running the task `gradle vagrantSetUp` which:
- collects all project archives dependencies and copies them into `${project.buildDir}/bats/archives`
- copy all project bats testing files from 'src/test/resources/packaging/tests' into `${project.buildDir}/bats/tests`
- copy all project bats utils files from 'src/test/resources/packaging/utils' into `${project.buildDir}/bats/utils`

It is also possible to inherit and grab the archives/tests/utils files from project dependencies using the plugin configuration:

apply plugin: 'elasticsearch.vagrant'
esvagrant {
    inheritTestUtils true|false
    inheritTestArchives true|false
    inheritTests true|false
}
dependencies {
    // Inherit Bats test utils from :qa:vagrant project
    bats project(path: ':qa:vagrant', configuration: 'bats')
}

The folders `${project.buildDir}/bats/archives`, `${project.buildDir}/bats/tests` and `${project.buildDir}/bats/utils` are then exported to the vagrant VMs and mapped to the BATS_ARCHIVES, BATS_TESTS and BATS_UTILS environnement variables.

The following Gradle tasks have also be renamed:

* gradle vagrantSetUp
This task copies all the necessary files to the project build directory (was `prepareTestRoot`)

* gradle vagrantSmokeTest
This task starts the VMs and echoes a "Hello world" within each VM (was: `smokeTest`)

TESTING.asciidoc

@@ -325,7 +325,7 @@ vagrant plugin install vagrant-cachier
 . Validate your installed dependencies:
 
 -------------------------------------
-gradle :qa:vagrant:checkVagrantVersion
+gradle :qa:vagrant:vagrantCheckVersion
 -------------------------------------
 
 . Download and smoke test the VMs with `gradle vagrantSmokeTest` or
@@ -417,17 +417,26 @@ and in another window:
 
 ----------------------------------------------------
 vagrant up centos-7 --provider virtualbox && vagrant ssh centos-7
-cd $TESTROOT
-sudo bats $BATS/*rpm*.bats
+cd $BATS_ARCHIVES
+sudo -E bats $BATS_TESTS/*rpm*.bats
 ----------------------------------------------------
 
 If you wanted to retest all the release artifacts on a single VM you could:
 
 -------------------------------------------------
-gradle prepareTestRoot
+gradle vagrantSetUp
 vagrant up ubuntu-1404 --provider virtualbox && vagrant ssh ubuntu-1404
-cd $TESTROOT
-sudo bats $BATS/*.bats
+cd $BATS_ARCHIVES
+sudo -E bats $BATS_TESTS/*.bats
+-------------------------------------------------
+
+Note: Starting vagrant VM outside of the elasticsearch folder requires to
+indicates the folder that contains the Vagrantfile using the VAGRANT_CWD
+environment variable:
+
+-------------------------------------------------
+gradle vagrantSetUp
+VAGRANT_CWD=/path/to/elasticsearch vagrant up centos-7 --provider virtualbox
 -------------------------------------------------
 
 == Coverage analysis

Vagrantfile

@@ -77,6 +77,9 @@ Vagrant.configure(2) do |config|
   # the elasticsearch project called vagrant....
   config.vm.synced_folder ".", "/vagrant", disabled: true
   config.vm.synced_folder ".", "/elasticsearch"
+  # Expose project directory
+  PROJECT_DIR = ENV['VAGRANT_PROJECT_DIR'] || Dir.pwd
+  config.vm.synced_folder PROJECT_DIR, "/project"
   config.vm.provider "virtualbox" do |v|
     # Give the boxes 3GB because Elasticsearch defaults to using 2GB
     v.memory = 3072
@@ -272,8 +275,10 @@ export ZIP=/elasticsearch/distribution/zip/build/distributions
 export TAR=/elasticsearch/distribution/tar/build/distributions
 export RPM=/elasticsearch/distribution/rpm/build/distributions
 export DEB=/elasticsearch/distribution/deb/build/distributions
-export TESTROOT=/elasticsearch/qa/vagrant/build/testroot
-export BATS=/elasticsearch/qa/vagrant/src/test/resources/packaging/scripts
+export BATS=/project/build/bats
+export BATS_UTILS=/project/build/bats/utils
+export BATS_TESTS=/project/build/bats/tests
+export BATS_ARCHIVES=/project/build/bats/archives
 VARS
   SHELL
 end

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/BatsOverVagrantTask.groovy

@@ -18,14 +18,7 @@
  */
 package org.elasticsearch.gradle.vagrant
 
-import org.gradle.api.DefaultTask
 import org.gradle.api.tasks.Input
-import org.gradle.api.tasks.TaskAction
-import org.gradle.logging.ProgressLoggerFactory
-import org.gradle.process.internal.ExecAction
-import org.gradle.process.internal.ExecActionFactory
-
-import javax.inject.Inject
 
 /**
  * Runs bats over vagrant. Pretty much like running it using Exec but with a

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantCommandTask.groovy

@@ -20,6 +20,7 @@ package org.elasticsearch.gradle.vagrant
 
 import org.apache.commons.io.output.TeeOutputStream
 import org.elasticsearch.gradle.LoggedExec
+import org.gradle.api.tasks.Exec
 import org.gradle.api.tasks.Input
 import org.gradle.logging.ProgressLoggerFactory
 
@@ -29,16 +30,23 @@ import javax.inject.Inject
  * Runs a vagrant command. Pretty much like Exec task but with a nicer output
  * formatter and defaults to `vagrant` as first part of commandLine.
  */
-public class VagrantCommandTask extends LoggedExec {
+public class VagrantCommandTask extends Exec {
 
     @Input
     String boxName
 
+    @Input
+    Map<String, String> environmentVars
+
     public VagrantCommandTask() {
         executable = 'vagrant'
+
         project.afterEvaluate {
             // It'd be nice if --machine-readable were, well, nice
             standardOutput = new TeeOutputStream(standardOutput, createLoggerOutputStream())
+            if (environmentVars != null) {
+                environment environmentVars
+            }
         }
     }
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantPropertiesExtension.groovy

@@ -0,0 +1,76 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.vagrant
+
+import org.gradle.api.tasks.Input
+
+class VagrantPropertiesExtension {
+
+    @Input
+    List<String> boxes
+
+    @Input
+    Long testSeed
+
+    @Input
+    String formattedTestSeed
+
+    @Input
+    String upgradeFromVersion
+
+    @Input
+    List<String> upgradeFromVersions
+
+    @Input
+    String batsDir
+
+    @Input
+    Boolean inheritTests
+
+    @Input
+    Boolean inheritTestArchives
+
+    @Input
+    Boolean inheritTestUtils
+
+    VagrantPropertiesExtension(List<String> availableBoxes) {
+        this.boxes = availableBoxes
+        this.batsDir = 'src/test/resources/packaging'
+    }
+
+    void boxes(String... boxes) {
+        this.boxes = Arrays.asList(boxes)
+    }
+
+    void setBatsDir(String batsDir) {
+        this.batsDir = batsDir
+    }
+
+    void setInheritTests(Boolean inheritTests) {
+        this.inheritTests = inheritTests
+    }
+
+    void setInheritTestArchives(Boolean inheritTestArchives) {
+        this.inheritTestArchives = inheritTestArchives
+    }
+
+    void setInheritTestUtils(Boolean inheritTestUtils) {
+        this.inheritTestUtils = inheritTestUtils
+    }
+}

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy

@@ -0,0 +1,438 @@
+package org.elasticsearch.gradle.vagrant
+
+import org.elasticsearch.gradle.FileContentsTask
+import org.elasticsearch.gradle.VersionProperties
+import org.gradle.BuildAdapter
+import org.gradle.BuildResult
+import org.gradle.api.GradleException
+import org.gradle.api.InvalidUserDataException
+import org.gradle.api.Plugin
+import org.gradle.api.Project
+import org.gradle.api.Task
+import org.gradle.api.artifacts.SelfResolvingDependency
+import org.gradle.api.artifacts.dsl.RepositoryHandler
+import org.gradle.api.internal.artifacts.dependencies.DefaultProjectDependency
+import org.gradle.api.internal.artifacts.dependencies.DefaultSelfResolvingDependency
+import org.gradle.api.tasks.Copy
+import org.gradle.api.tasks.Delete
+import org.gradle.api.tasks.Exec
+
+class VagrantTestPlugin implements Plugin<Project> {
+
+    /** All available boxes **/
+    static List<String> BOXES = [
+            'centos-6',
+            'centos-7',
+            'debian-8',
+            'fedora-24',
+            'oel-6',
+            'oel-7',
+            'opensuse-13',
+            'sles-12',
+            'ubuntu-1204',
+            'ubuntu-1404',
+            'ubuntu-1604'
+    ]
+
+    /** Boxes used when sampling the tests **/
+    static List<String> SAMPLE = [
+            'centos-7',
+            'ubuntu-1404',
+    ]
+
+    /** All onboarded archives by default, available for Bats tests even if not used **/
+    static List<String> DISTRIBUTION_ARCHIVES = ['tar', 'rpm', 'deb']
+
+    /** Packages onboarded for upgrade tests **/
+    static List<String> UPGRADE_FROM_ARCHIVES = ['rpm', 'deb']
+
+    private static final BATS = 'bats'
+    private static final String BATS_TEST_COMMAND ="cd \$BATS_ARCHIVES && sudo -E bats --tap \$BATS_TESTS/*.$BATS"
+
+    @Override
+    void apply(Project project) {
+
+        // Creates the Vagrant extension for the project
+        project.extensions.create('esvagrant', VagrantPropertiesExtension, listVagrantBoxes(project))
+
+        // Add required repositories for Bats tests
+        configureBatsRepositories(project)
+
+        // Creates custom configurations for Bats testing files (and associated scripts and archives)
+        createBatsConfiguration(project)
+
+        // Creates all the main Vagrant tasks
+        createVagrantTasks(project)
+
+        if (project.extensions.esvagrant.boxes == null || project.extensions.esvagrant.boxes.size() == 0) {
+            throw new InvalidUserDataException('Vagrant boxes cannot be null or empty for esvagrant')
+        }
+
+        for (String box : project.extensions.esvagrant.boxes) {
+            if (BOXES.contains(box) == false) {
+                throw new InvalidUserDataException("Vagrant box [${box}] not found, available virtual machines are ${BOXES}")
+            }
+        }
+
+        // Creates all tasks related to the Vagrant boxes
+        createVagrantBoxesTasks(project)
+    }
+
+    private List<String> listVagrantBoxes(Project project) {
+        String vagrantBoxes = project.getProperties().get('vagrant.boxes', 'sample')
+        if (vagrantBoxes == 'sample') {
+            return SAMPLE
+        } else if (vagrantBoxes == 'all') {
+            return BOXES
+        } else {
+            return vagrantBoxes.split(',')
+        }
+    }
+
+    private static Set<String> listVersions(Project project) {
+        Node xml
+        new URL('https://repo1.maven.org/maven2/org/elasticsearch/elasticsearch/maven-metadata.xml').openStream().withStream { s ->
+            xml = new XmlParser().parse(s)
+        }
+        Set<String> versions = new TreeSet<>(xml.versioning.versions.version.collect { it.text() }.findAll { it ==~ /[5]\.\d\.\d/ })
+        if (versions.isEmpty() == false) {
+            return versions;
+        }
+
+        // If no version is found, we run the tests with the current version
+        return Collections.singleton(project.version);
+    }
+
+    private static File getVersionsFile(Project project) {
+        File versions = new File(project.projectDir, 'versions');
+        if (versions.exists() == false) {
+            // Use the elasticsearch's versions file from project :qa:vagrant
+            versions = project.project(":qa:vagrant").file('versions')
+        }
+        return versions
+    }
+
+    private static void configureBatsRepositories(Project project) {
+        RepositoryHandler repos = project.repositories
+
+        // Try maven central first, it'll have releases before 5.0.0
+        repos.mavenCentral()
+
+        /* Setup a repository that tries to download from
+          https://artifacts.elastic.co/downloads/elasticsearch/[module]-[revision].[ext]
+          which should work for 5.0.0+. This isn't a real ivy repository but gradle
+          is fine with that */
+        repos.ivy {
+            artifactPattern "https://artifacts.elastic.co/downloads/elasticsearch/[module]-[revision].[ext]"
+        }
+    }
+
+    private static void createBatsConfiguration(Project project) {
+        project.configurations.create(BATS)
+
+        Long seed
+        String formattedSeed = null
+        String[] upgradeFromVersions
+
+        String maybeTestsSeed = System.getProperty("tests.seed", null);
+        if (maybeTestsSeed != null) {
+            List<String> seeds = maybeTestsSeed.tokenize(':')
+            if (seeds.size() != 0) {
+                String masterSeed = seeds.get(0)
+                seed = new BigInteger(masterSeed, 16).longValue()
+                formattedSeed = maybeTestsSeed
+            }
+        }
+        if (formattedSeed == null) {
+            seed = new Random().nextLong()
+            formattedSeed = String.format("%016X", seed)
+        }
+
+        String maybeUpdradeFromVersions = System.getProperty("tests.packaging.upgrade.from.versions", null)
+        if (maybeUpdradeFromVersions != null) {
+            upgradeFromVersions = maybeUpdradeFromVersions.split(",")
+        } else {
+            upgradeFromVersions = getVersionsFile(project)
+        }
+
+        String upgradeFromVersion = upgradeFromVersions[new Random(seed).nextInt(upgradeFromVersions.length)]
+
+        DISTRIBUTION_ARCHIVES.each {
+            // Adds a dependency for the current version
+            project.dependencies.add(BATS, project.dependencies.project(path: ":distribution:${it}", configuration: 'archives'))
+        }
+
+        UPGRADE_FROM_ARCHIVES.each {
+            // The version of elasticsearch that we upgrade *from*
+            project.dependencies.add(BATS, "org.elasticsearch.distribution.${it}:elasticsearch:${upgradeFromVersion}@${it}")
+        }
+
+        project.extensions.esvagrant.testSeed = seed
+        project.extensions.esvagrant.formattedTestSeed = formattedSeed
+        project.extensions.esvagrant.upgradeFromVersion = upgradeFromVersion
+        project.extensions.esvagrant.upgradeFromVersions = upgradeFromVersions
+    }
+
+    private static void createCleanTask(Project project) {
+        project.tasks.create('clean', Delete.class) {
+            description 'Clean the project build directory'
+            group 'Build'
+            delete project.buildDir
+        }
+    }
+
+    private static void createStopTask(Project project) {
+        project.tasks.create('stop') {
+            description 'Stop any tasks from tests that still may be running'
+            group 'Verification'
+        }
+    }
+
+    private static void createSmokeTestTask(Project project) {
+        project.tasks.create('vagrantSmokeTest') {
+            description 'Smoke test the specified vagrant boxes'
+            group 'Verification'
+        }
+    }
+
+    private static void createPrepareVagrantTestEnvTask(Project project) {
+        File batsDir = new File("${project.buildDir}/${BATS}")
+
+        Task createBatsDirsTask = project.tasks.create('createBatsDirs')
+        createBatsDirsTask.outputs.dir batsDir
+        createBatsDirsTask.dependsOn project.tasks.vagrantVerifyVersions
+        createBatsDirsTask.doLast {
+            batsDir.mkdirs()
+        }
+
+        Copy copyBatsArchives = project.tasks.create('copyBatsArchives', Copy) {
+            dependsOn createBatsDirsTask
+            into "${batsDir}/archives"
+            from project.configurations[BATS]
+        }
+
+        Copy copyBatsTests = project.tasks.create('copyBatsTests', Copy) {
+            dependsOn createBatsDirsTask
+            into "${batsDir}/tests"
+            from {
+                "${project.extensions.esvagrant.batsDir}/tests"
+            }
+        }
+
+        Copy copyBatsUtils = project.tasks.create('copyBatsUtils', Copy) {
+            dependsOn createBatsDirsTask
+            into "${batsDir}/utils"
+            from {
+                "${project.extensions.esvagrant.batsDir}/utils"
+            }
+        }
+
+        // Now we iterate over dependencies of the bats configuration. When a project dependency is found,
+        // we bring back its own archives, test files or test utils.
+        project.afterEvaluate {
+            project.configurations.bats.dependencies.findAll {it.configuration == BATS }.each { d ->
+                if (d instanceof DefaultProjectDependency) {
+                    DefaultProjectDependency externalBatsDependency = (DefaultProjectDependency) d
+                    Project externalBatsProject = externalBatsDependency.dependencyProject
+                    String externalBatsDir = externalBatsProject.extensions.esvagrant.batsDir
+
+                    if (project.extensions.esvagrant.inheritTests) {
+                        copyBatsTests.from(externalBatsProject.files("${externalBatsDir}/tests"))
+                    }
+                    if (project.extensions.esvagrant.inheritTestArchives) {
+                        copyBatsArchives.from(externalBatsDependency.projectConfiguration.files)
+                    }
+                    if (project.extensions.esvagrant.inheritTestUtils) {
+                        copyBatsUtils.from(externalBatsProject.files("${externalBatsDir}/utils"))
+                    }
+                }
+            }
+        }
+
+        Task createVersionFile = project.tasks.create('createVersionFile', FileContentsTask) {
+            dependsOn createBatsDirsTask
+            file "${batsDir}/archives/version"
+            contents project.version
+        }
+
+        Task createUpgradeFromFile = project.tasks.create('createUpgradeFromFile', FileContentsTask) {
+            dependsOn createBatsDirsTask
+            file "${batsDir}/archives/upgrade_from_version"
+            contents project.extensions.esvagrant.upgradeFromVersion
+        }
+
+        Task vagrantSetUpTask = project.tasks.create('vagrantSetUp')
+        vagrantSetUpTask.dependsOn 'vagrantCheckVersion'
+        vagrantSetUpTask.dependsOn copyBatsTests, copyBatsUtils, copyBatsArchives, createVersionFile, createUpgradeFromFile
+        vagrantSetUpTask.doFirst {
+            project.gradle.addBuildListener new BuildAdapter() {
+                @Override
+                void buildFinished(BuildResult result) {
+                    if (result.failure) {
+                        println "Reproduce with: gradle packagingTest "
+                        +"-Pvagrant.boxes=${project.extensions.esvagrant.boxes} "
+                        + "-Dtests.seed=${project.extensions.esvagrant.formattedSeed} "
+                        + "-Dtests.packaging.upgrade.from.versions=${project.extensions.esvagrant.upgradeFromVersions.join(",")}"
+                    }
+                }
+            }
+        }
+    }
+
+    private static void createUpdateVersionsTask(Project project) {
+        project.tasks.create('vagrantUpdateVersions') {
+            description 'Update file containing options for the\n    "starting" version in the "upgrade from" packaging tests.'
+            group 'Verification'
+            doLast {
+                File versions = getVersionsFile(project)
+                versions.text = listVersions(project).join('\n') + '\n'
+            }
+        }
+    }
+
+    private static void createVerifyVersionsTask(Project project) {
+        project.tasks.create('vagrantVerifyVersions') {
+            description 'Update file containing options for the\n    "starting" version in the "upgrade from" packaging tests.'
+            group 'Verification'
+            doLast {
+                String maybeUpdateFromVersions = System.getProperty("tests.packaging.upgrade.from.versions", null)
+                if (maybeUpdateFromVersions == null) {
+                    Set<String> versions = listVersions(project)
+                    Set<String> actualVersions = new TreeSet<>(project.extensions.esvagrant.upgradeFromVersions)
+                    if (!versions.equals(actualVersions)) {
+                        throw new GradleException("out-of-date versions " + actualVersions +
+                                ", expected " + versions + "; run gradle vagrantUpdateVersions")
+                    }
+                }
+            }
+        }
+    }
+
+    private static void createCheckVersionTask(Project project) {
+        project.tasks.create('vagrantCheckVersion', Exec) {
+            description 'Check the Vagrant version'
+            group 'Verification'
+            commandLine 'vagrant', '--version'
+            standardOutput = new ByteArrayOutputStream()
+            doLast {
+                String version = standardOutput.toString().trim()
+                if ((version ==~ /Vagrant 1\.[789]\..+/) == false) {
+                    throw new InvalidUserDataException("Illegal version of vagrant [${version}]. Need [Vagrant 1.7+]")
+                }
+            }
+        }
+    }
+
+    private static void createPackagingTestTask(Project project) {
+        project.tasks.create('packagingTest') {
+            group 'Verification'
+            description "Tests yum/apt packages using vagrant and bats.\n" +
+                    "    Specify the vagrant boxes to test using the gradle property 'vagrant.boxes'.\n" +
+                    "    'sample' can be used to test a single yum and apt box. 'all' can be used to\n" +
+                    "    test all available boxes. The available boxes are: \n" +
+                    "    ${BOXES}"
+            dependsOn 'vagrantCheckVersion'
+        }
+    }
+
+    private static void createVagrantTasks(Project project) {
+        createCleanTask(project)
+        createStopTask(project)
+        createSmokeTestTask(project)
+        createUpdateVersionsTask(project)
+        createVerifyVersionsTask(project)
+        createCheckVersionTask(project)
+        createPrepareVagrantTestEnvTask(project)
+        createPackagingTestTask(project)
+    }
+
+    private static void createVagrantBoxesTasks(Project project) {
+        assert project.extensions.esvagrant.boxes != null
+
+        assert project.tasks.stop != null
+        Task stop = project.tasks.stop
+
+        assert project.tasks.vagrantSmokeTest != null
+        Task vagrantSmokeTest = project.tasks.vagrantSmokeTest
+
+        assert project.tasks.vagrantCheckVersion != null
+        Task vagrantCheckVersion = project.tasks.vagrantCheckVersion
+
+        assert project.tasks.vagrantSetUp != null
+        Task vagrantSetUp = project.tasks.vagrantSetUp
+
+        assert project.tasks.packagingTest != null
+        Task packagingTest = project.tasks.packagingTest
+
+        /*
+         * We always use the main project.rootDir as Vagrant's current working directory (VAGRANT_CWD)
+         * so that boxes are not duplicated for every Gradle project that use this VagrantTestPlugin.
+         */
+        def vagrantEnvVars = [
+                'VAGRANT_CWD'           : "${project.rootDir.absolutePath}",
+                'VAGRANT_VAGRANTFILE'   : 'Vagrantfile',
+                'VAGRANT_PROJECT_DIR'   : "${project.projectDir.absolutePath}"
+        ]
+
+        // Each box gets it own set of tasks
+        for (String box : BOXES) {
+            String boxTask = box.capitalize().replace('-', '')
+
+            // always add a halt task for all boxes, so clean makes sure they are all shutdown
+            Task halt = project.tasks.create("vagrant${boxTask}#halt", VagrantCommandTask) {
+                boxName box
+                environmentVars vagrantEnvVars
+                args 'halt', box
+            }
+            stop.dependsOn(halt)
+            if (project.extensions.esvagrant.boxes.contains(box) == false) {
+                // we only need a halt task if this box was not specified
+                continue;
+            }
+
+            Task update = project.tasks.create("vagrant${boxTask}#update", VagrantCommandTask) {
+                boxName box
+                environmentVars vagrantEnvVars
+                args 'box', 'update', box
+                dependsOn vagrantCheckVersion, vagrantSetUp
+            }
+
+            Task up = project.tasks.create("vagrant${boxTask}#up", VagrantCommandTask) {
+                boxName box
+                environmentVars vagrantEnvVars
+                /* Its important that we try to reprovision the box even if it already
+                  exists. That way updates to the vagrant configuration take automatically.
+                  That isn't to say that the updates will always be compatible. Its ok to
+                  just destroy the boxes if they get busted but that is a manual step
+                  because its slow-ish. */
+                /* We lock the provider to virtualbox because the Vagrantfile specifies
+                  lots of boxes that only work properly in virtualbox. Virtualbox is
+                  vagrant's default but its possible to change that default and folks do.
+                  But the boxes that we use are unlikely to work properly with other
+                  virtualization providers. Thus the lock. */
+                args 'up', box, '--provision', '--provider', 'virtualbox'
+                /* It'd be possible to check if the box is already up here and output
+                  SKIPPED but that would require running vagrant status which is slow! */
+                dependsOn update
+            }
+
+            Task smoke = project.tasks.create("vagrant${boxTask}#smoketest", Exec) {
+                environment vagrantEnvVars
+                dependsOn up
+                finalizedBy halt
+                commandLine 'vagrant', 'ssh', box, '--command',
+                        "set -o pipefail && echo 'Hello from ${project.path}' | sed -ue 's/^/    ${box}: /'"
+            }
+            vagrantSmokeTest.dependsOn(smoke)
+
+            Task packaging = project.tasks.create("vagrant${boxTask}#packagingtest", BatsOverVagrantTask) {
+                boxName box
+                environmentVars vagrantEnvVars
+                dependsOn up
+                finalizedBy halt
+                command BATS_TEST_COMMAND
+            }
+            packagingTest.dependsOn(packaging)
+        }
+    }
+}

buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.vagrant.properties

@@ -0,0 +1 @@
+implementation-class=org.elasticsearch.gradle.vagrant.VagrantTestPlugin

qa/vagrant/build.gradle

@@ -17,288 +17,13 @@
  * under the License.
  */
 
-import org.elasticsearch.gradle.FileContentsTask
-import org.elasticsearch.gradle.vagrant.BatsOverVagrantTask
-import org.elasticsearch.gradle.vagrant.VagrantCommandTask
-
-String testScripts = '*.bats'
-String testCommand = "cd \$TESTROOT && sudo bats --tap \$BATS/$testScripts"
-String smokeTestCommand = 'echo I work'
-
-// the images we allow testing with
-List<String> availableBoxes = [
-    'centos-6',
-    'centos-7',
-    'debian-8',
-    'fedora-24',
-    'oel-6',
-    'oel-7',
-    'opensuse-13',
-    'sles-12',
-    'ubuntu-1204',
-    'ubuntu-1404',
-    'ubuntu-1604'
-]
-
-String vagrantBoxes = getProperties().get('vagrant.boxes', 'sample')
-List<String> boxes = []
-for (String box : vagrantBoxes.split(',')) {
-  if (box == 'sample') {
-    boxes.add('centos-7')
-    boxes.add('ubuntu-1404')
-  } else if (box == 'all') {
-    boxes = availableBoxes
-    break
-  } else {
-    if (availableBoxes.contains(box) == false) {
-      throw new IllegalArgumentException("Unknown vagrant box '${box}'")
-    }
-    boxes.add(box)
-  }
-}
-
-long seed
-String formattedSeed = null
-String[] upgradeFromVersions
-String upgradeFromVersion
-
-String maybeTestsSeed = System.getProperty("tests.seed", null);
-if (maybeTestsSeed != null) {
-  List<String> seeds = maybeTestsSeed.tokenize(':')
-  if (seeds.size() != 0) {
-    String masterSeed = seeds.get(0)
-    seed = new BigInteger(masterSeed, 16).longValue()
-    formattedSeed = maybeTestsSeed
-  }
-}
-if (formattedSeed == null) {
-  seed = new Random().nextLong()
-  formattedSeed = String.format("%016X", seed)
-}
-
-String maybeUpdradeFromVersions = System.getProperty("tests.packaging.upgrade.from.versions", null)
-if (maybeUpdradeFromVersions != null) {
-  upgradeFromVersions = maybeUpdradeFromVersions.split(",")
-} else {
-  upgradeFromVersions = new File(project.projectDir, 'versions')
-}
-
-upgradeFromVersion = upgradeFromVersions[new Random(seed).nextInt(upgradeFromVersions.length)]
-
-configurations {
-  test
-}
-
-repositories {
-  mavenCentral() // Try maven central first, it'll have releases before 5.0.0
-  /* Setup a repository that tries to download from
-    https://artifacts.elastic.co/downloads/elasticsearch/[module]-[revision].[ext]
-    which should work for 5.0.0+. This isn't a real ivy repository but gradle
-    is fine with that */
-  ivy {
-    artifactPattern "https://artifacts.elastic.co/downloads/elasticsearch/[module]-[revision].[ext]"
-  }
-}
+apply plugin: 'elasticsearch.vagrant'
 
 dependencies {
-  test project(path: ':distribution:tar', configuration: 'archives')
-  test project(path: ':distribution:rpm', configuration: 'archives')
-  test project(path: ':distribution:deb', configuration: 'archives')
-
   // Collect all the plugins
   for (Project subproj : project.rootProject.subprojects) {
     if (subproj.path.startsWith(':plugins:')) {
-      test project(path: "${subproj.path}", configuration: 'zip')
-    }
-  }
-
-  // The version of elasticsearch that we upgrade *from*
-  test "org.elasticsearch.distribution.deb:elasticsearch:$upgradeFromVersion@deb"
-  test "org.elasticsearch.distribution.rpm:elasticsearch:$upgradeFromVersion@rpm"
-}
-
-task clean(type: Delete) {
-  group 'Build'
-  delete buildDir
-}
-
-task stop {
-  group 'Verification'
-  description 'Stop any tasks from tests that still may be running'
-}
-
-Set<String> getVersions() {
-  Node xml
-  new URL('https://repo1.maven.org/maven2/org/elasticsearch/elasticsearch/maven-metadata.xml').openStream().withStream { s ->
-    xml = new XmlParser().parse(s)
-  }
-  Set<String> versions = new TreeSet<>(xml.versioning.versions.version.collect { it.text() }.findAll { it ==~ /[5]\.\d\.\d/ })
-  if (versions.isEmpty() == false) {
-    return versions;
-  }
-
-  // If no version is found, we run the tests with the current version
-  return Collections.singleton(project.version);
-}
-
-task updatePackagingTestUpgradeFromVersions {
-  group 'Verification'
-  description 'Update file containing options for the\n    "starting" version in the "upgrade from" packaging tests.'
-  doLast {
-    Set<String> versions = getVersions()
-    new File(project.projectDir, 'versions').text = versions.join('\n') + '\n'
-  }
-}
-
-task verifyPackagingTestUpgradeFromVersions {
-  doLast {
-    String maybeUpdateFromVersions = System.getProperty("tests.packaging.upgrade.from.versions", null)
-    if (maybeUpdateFromVersions == null) {
-      Set<String> versions = getVersions()
-      Set<String> actualVersions = new TreeSet<>(Arrays.asList(upgradeFromVersions))
-      if (!versions.equals(actualVersions)) {
-        throw new GradleException("out-of-date versions " + actualVersions +
-          ", expected " + versions + "; run gradle updatePackagingTestUpgradeFromVersions")
-      }
+      bats project(path: "${subproj.path}", configuration: 'zip')
     }
   }
 }
-
-File testRoot = new File("$buildDir/testroot")
-task createTestRoot {
-  dependsOn verifyPackagingTestUpgradeFromVersions
-  outputs.dir testRoot
-  doLast {
-    testRoot.mkdirs()
-  }
-}
-
-task createVersionFile(type: FileContentsTask) {
-  dependsOn createTestRoot
-  file "${testRoot}/version"
-  contents = version
-}
-
-task createUpgradeFromFile(type: FileContentsTask) {
-  dependsOn createTestRoot
-  file "${testRoot}/upgrade_from_version"
-  contents = upgradeFromVersion
-}
-
-task prepareTestRoot(type: Copy) {
-  description 'Dump bats test dependencies into the $TESTROOT'
-  into testRoot
-  from configurations.test
-
-  dependsOn createVersionFile, createUpgradeFromFile
-  doFirst {
-    gradle.addBuildListener new BuildAdapter() {
-      @Override
-      void buildFinished(BuildResult result) {
-        if (result.failure) {
-          println "Reproduce with: gradle packagingTest -Pvagrant.boxes=${vagrantBoxes} -Dtests.seed=${formattedSeed} -Dtests.packaging.upgrade.from.versions=${upgradeFromVersions.join(",")}"
-        }
-      }
-    }
-  }
-}
-
-task checkVagrantVersion(type: Exec) {
-  commandLine 'vagrant', '--version'
-  standardOutput = new ByteArrayOutputStream()
-  doLast {
-    String version = standardOutput.toString().trim()
-    if ((version ==~ /Vagrant 1\.(8\.[6-9]|9\.[0-9])+/) == false) {
-      throw new InvalidUserDataException("Illegal version of vagrant [${version}]. Need [Vagrant 1.8.6+]")
-    }
-  }
-}
-
-task checkVirtualBoxVersion(type: Exec) {
-  commandLine 'vboxmanage', '--version'
-  standardOutput = new ByteArrayOutputStream()
-  doLast {
-    String version = standardOutput.toString().trim()
-    try {
-      String[] versions = version.split('\\.')
-      int major = Integer.parseInt(versions[0])
-      int minor = Integer.parseInt(versions[1])
-      if ((major < 5) || (major == 5 && minor < 1)) {
-        throw new InvalidUserDataException("Illegal version of virtualbox [${version}]. Need [5.1+]")
-      }
-    } catch (NumberFormatException | ArrayIndexOutOfBoundsException e) {
-      throw new InvalidUserDataException("Unable to parse version of virtualbox [${version}]. Required [5.1+]", e)
-    }
-  }
-}
-
-task vagrantSmokeTest {
-  group 'Verification'
-  description 'Smoke test the specified vagrant boxes'
-}
-
-task packagingTest {
-  group 'Verification'
-  description "Tests yum/apt packages using vagrant and bats.\n" +
-              "    Specify the vagrant boxes to test using the gradle property 'vagrant.boxes'.\n" +
-              "    'sample' can be used to test a single yum and apt box. 'all' can be used to\n" +
-              "    test all available boxes. The available boxes are: \n" +
-              "    ${availableBoxes}"
-}
-
-// Each box gets it own set of tasks
-for (String box : availableBoxes) {
-  String boxTask = box.capitalize().replace('-', '')
-
-  // always add a halt task for all boxes, so clean makes sure they are all shutdown
-  Task halt = tasks.create("vagrant${boxTask}#halt", VagrantCommandTask) {
-    boxName box
-    args 'halt', box
-  }
-  stop.dependsOn(halt)
-  if (boxes.contains(box) == false) {
-    // we only need a halt task if this box was not specified
-    continue;
-  }
-
-  Task update = tasks.create("vagrant${boxTask}#update", VagrantCommandTask) {
-    boxName box
-    args 'box', 'update', box
-    dependsOn checkVagrantVersion, checkVirtualBoxVersion
-  }
-
-  Task up = tasks.create("vagrant${boxTask}#up", VagrantCommandTask) {
-    boxName box
-    /* It's important that we try to reprovision the box even if it already
-      exists. That way updates to the vagrant configuration take automatically.
-      That isn't to say that the updates will always be compatible. Its ok to
-      just destroy the boxes if they get busted but that is a manual step
-      because its slow-ish. */
-    /* We lock the provider to virtualbox because the Vagrantfile specifies
-      lots of boxes that only work properly in virtualbox. Virtualbox is
-      vagrant's default but its possible to change that default and folks do.
-      But the boxes that we use are unlikely to work properly with other
-      virtualization providers. Thus the lock. */
-    args 'up', box, '--provision', '--provider', 'virtualbox'
-    /* It'd be possible to check if the box is already up here and output
-      SKIPPED but that would require running vagrant status which is slow! */
-    dependsOn update
-  }
-
-  Task smoke = tasks.create("vagrant${boxTask}#smoketest", Exec) {
-    dependsOn up
-    finalizedBy halt
-    commandLine 'vagrant', 'ssh', box, '--command',
-      "set -o pipefail && ${smokeTestCommand} | sed -ue 's/^/    ${box}: /'"
-  }
-  vagrantSmokeTest.dependsOn(smoke)
-
-  Task packaging = tasks.create("packagingTest${boxTask}", BatsOverVagrantTask) {
-    dependsOn up
-    finalizedBy halt
-    boxName box
-    command testCommand
-    dependsOn prepareTestRoot
-  }
-  packagingTest.dependsOn(packaging)
-}

qa/vagrant/src/test/resources/packaging/scripts/20_tar_package.bats → qa/vagrant/src/test/resources/packaging/tests/20_tar_package.bats

@@ -29,9 +29,9 @@
 # under the License.
 
 # Load test utilities
-load packaging_test_utils
-load tar
-load plugins
+load $BATS_UTILS/utils.bash
+load $BATS_UTILS/tar.bash
+load $BATS_UTILS/plugins.bash
 
 setup() {
     skip_not_tar_gz

qa/vagrant/src/test/resources/packaging/scripts/30_deb_package.bats → qa/vagrant/src/test/resources/packaging/tests/30_deb_package.bats

@@ -30,9 +30,9 @@
 # under the License.
 
 # Load test utilities
-load packaging_test_utils
-load os_package
-load plugins
+load $BATS_UTILS/utils.bash
+load $BATS_UTILS/packages.bash
+load $BATS_UTILS/plugins.bash
 
 # Cleans everything for the 1st execution
 setup() {

qa/vagrant/src/test/resources/packaging/scripts/40_rpm_package.bats → qa/vagrant/src/test/resources/packaging/tests/40_rpm_package.bats

@@ -29,9 +29,9 @@
 # under the License.
 
 # Load test utilities
-load packaging_test_utils
-load os_package
-load plugins
+load $BATS_UTILS/utils.bash
+load $BATS_UTILS/packages.bash
+load $BATS_UTILS/plugins.bash
 
 # Cleans everything for the 1st execution
 setup() {

qa/vagrant/src/test/resources/packaging/scripts/60_systemd.bats → qa/vagrant/src/test/resources/packaging/tests/60_systemd.bats

@@ -29,9 +29,9 @@
 # under the License.
 
 # Load test utilities
-load packaging_test_utils
-load os_package
-load plugins
+load $BATS_UTILS/utils.bash
+load $BATS_UTILS/packages.bash
+load $BATS_UTILS/plugins.bash
 
 # Cleans everything for the 1st execution
 setup() {

qa/vagrant/src/test/resources/packaging/scripts/70_sysv_initd.bats → qa/vagrant/src/test/resources/packaging/tests/70_sysv_initd.bats

@@ -29,9 +29,9 @@
 # under the License.
 
 # Load test utilities
-load packaging_test_utils
-load os_package
-load plugins
+load $BATS_UTILS/utils.bash
+load $BATS_UTILS/packages.bash
+load $BATS_UTILS/plugins.bash
 
 # Cleans everything for the 1st execution
 setup() {

qa/vagrant/src/test/resources/packaging/scripts/80_upgrade.bats → qa/vagrant/src/test/resources/packaging/tests/80_upgrade.bats

@@ -31,8 +31,8 @@
 # under the License.
 
 # Load test utilities
-load packaging_test_utils
-load os_package
+load $BATS_UTILS/utils.bash
+load $BATS_UTILS/packages.bash
 
 # Cleans everything for the 1st execution
 setup() {

qa/vagrant/src/test/resources/packaging/scripts/90_reinstall.bats → qa/vagrant/src/test/resources/packaging/tests/90_reinstall.bats

@@ -31,8 +31,8 @@
 # under the License.
 
 # Load test utilities
-load packaging_test_utils
-load os_package
+load $BATS_UTILS/utils.bash
+load $BATS_UTILS/packages.bash
 
 # Cleans everything for the 1st execution
 setup() {

qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash → qa/vagrant/src/test/resources/packaging/tests/module_and_plugin_test_cases.bash

@@ -39,9 +39,9 @@
 # system uses.
 
 # Load test utilities
-load packaging_test_utils
-load modules
-load plugins
+load $BATS_UTILS/utils.bash
+load $BATS_UTILS/modules.bash
+load $BATS_UTILS/plugins.bash
 
 setup() {
     # The rules on when we should clean an reinstall are complex - all the
@@ -60,7 +60,7 @@ setup() {
 }
 
 if [[ "$BATS_TEST_FILENAME" =~ 25_tar_plugins.bats$ ]]; then
-    load tar
+    load $BATS_UTILS/tar.bash
     GROUP='TAR PLUGINS'
     install() {
         install_archive
@@ -70,7 +70,7 @@ if [[ "$BATS_TEST_FILENAME" =~ 25_tar_plugins.bats$ ]]; then
     export_elasticsearch_paths
     export ESPLUGIN_COMMAND_USER=elasticsearch
 else
-    load os_package
+    load $BATS_UTILS/packages.bash
     if is_rpm; then
         GROUP='RPM PLUGINS'
     elif is_dpkg; then