Tests: Add platformTest to vagrant (#23339)

This change adds a new test task, platformTest, which runs `gradle test
integTest` within a vagrant host. This will allow us to still test on
all the supported platforms, but be able to standardize on the tools
provided in the host system, for example, with a modern version of git
that can allow #22946.

In order to have sufficient memory and cpu to run the tests, the
vagrantfile has been updated to use 8GB and 4 cpus by default. This can
be customized with the `VAGRANT_MEMORY` and `VAGRANT_CPUS` environment
variables.  Also, to save time to show this can work, it currently uses
the same Vagrantfile the packaging tests do. There are a lot of cleanups
we can do to how the gradle-vagrant tasks work, including generating
Vagrantfile altogether, but I think this is fine for now as the same
machines that would run platformTest run packagingTest, and they are
relatively beefy machines, so the higher memory and cpu for them, with
either task, should not be an issue.

Vagrantfile

@@ -193,7 +193,15 @@ def provision(config,
   raise ArgumentError.new('update_command is required') if update_command == 'required'
   raise ArgumentError.new('update_tracking_file is required') if update_tracking_file == 'required'
   raise ArgumentError.new('install_command is required') if install_command == 'required'
-  config.vm.provision "bats dependencies", type: "shell", inline: <<-SHELL
+  config.vm.provider "virtualbox" do |v|
+    # Give the box more memory and cpu because our tests are beasts!
+    v.memory = Integer(ENV['VAGRANT_MEMORY'] || 8192)
+    v.cpus = Integer(ENV['VAGRANT_CPUS'] || 4)
+  end
+  config.vm.synced_folder "#{Dir.home}/.gradle/caches", "/home/vagrant/.gradle/caches",
+    create: true,
+    owner: "vagrant"
+  config.vm.provision "dependencies", type: "shell", inline: <<-SHELL
     set -e
     set -o pipefail
 
@@ -256,6 +264,18 @@ def provision(config,
       /tmp/bats/install.sh /usr
       rm -rf /tmp/bats
     }
+
+    installed gradle || {
+      echo "==> Installing gradle"
+      curl -o /tmp/gradle.zip -L https://services.gradle.org/distributions/gradle-3.3-bin.zip
+      unzip /tmp/gradle.zip -d /opt
+      rm -rf /tmp/gradle.zip 
+      ln -s /opt/gradle-3.3/bin/gradle /usr/bin/gradle
+      # make nfs mounted gradle home dir writeable
+      chown vagrant:vagrant /home/vagrant/.gradle
+    }
+
+
     cat \<\<VARS > /etc/profile.d/elasticsearch_vars.sh
 export ZIP=/elasticsearch/distribution/zip/build/distributions
 export TAR=/elasticsearch/distribution/tar/build/distributions
@@ -265,6 +285,7 @@ export BATS=/project/build/bats
 export BATS_UTILS=/project/build/bats/utils
 export BATS_TESTS=/project/build/bats/tests
 export BATS_ARCHIVES=/project/build/bats/archives
+export GRADLE_HOME=/opt/gradle-3.3
 VARS
     cat \<\<SUDOERS_VARS > /etc/sudoers.d/elasticsearch_vars
 Defaults   env_keep += "ZIP"

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy

@@ -41,6 +41,7 @@ class VagrantTestPlugin implements Plugin<Project> {
 
     private static final BATS = 'bats'
     private static final String BATS_TEST_COMMAND ="cd \$BATS_ARCHIVES && sudo bats --tap \$BATS_TESTS/*.$BATS"
+    private static final String PLATFORM_TEST_COMMAND ="rm -rf ~/elasticsearch && rsync -r /elasticsearch/ ~/elasticsearch && cd ~/elasticsearch && \$GRADLE_HOME/bin/gradle test integTest"
 
     @Override
     void apply(Project project) {
@@ -327,6 +328,17 @@ class VagrantTestPlugin implements Plugin<Project> {
         }
     }
 
+    private static void createPlatformTestTask(Project project) {
+        project.tasks.create('platformTest') {
+            group 'Verification'
+            description "Test unit and integ tests on different platforms using vagrant.\n" +
+                    "    Specify the vagrant boxes to test using the gradle property 'vagrant.boxes'.\n" +
+                    "    'all' can be used to test all available boxes. The available boxes are: \n" +
+                    "    ${BOXES}"
+            dependsOn 'vagrantCheckVersion'
+        }
+    }
+
     private static void createVagrantTasks(Project project) {
         createCleanTask(project)
         createStopTask(project)
@@ -337,6 +349,7 @@ class VagrantTestPlugin implements Plugin<Project> {
         createCheckVirtualBoxVersionTask(project)
         createPrepareVagrantTestEnvTask(project)
         createPackagingTestTask(project)
+        createPlatformTestTask(project)
     }
 
     private static void createVagrantBoxesTasks(Project project) {
@@ -360,6 +373,9 @@ class VagrantTestPlugin implements Plugin<Project> {
         assert project.tasks.packagingTest != null
         Task packagingTest = project.tasks.packagingTest
 
+        assert project.tasks.platformTest != null
+        Task platformTest = project.tasks.platformTest
+
         /*
          * We always use the main project.rootDir as Vagrant's current working directory (VAGRANT_CWD)
          * so that boxes are not duplicated for every Gradle project that use this VagrantTestPlugin.
@@ -425,7 +441,8 @@ class VagrantTestPlugin implements Plugin<Project> {
                 finalizedBy halt
                 command BATS_TEST_COMMAND
             }
-            TaskExecutionAdapter reproduceListener = new TaskExecutionAdapter() {
+
+            TaskExecutionAdapter packagingReproListener = new TaskExecutionAdapter() {
                 @Override
                 void afterExecute(Task task, TaskState state) {
                     if (state.failure != null) {
@@ -435,15 +452,40 @@ class VagrantTestPlugin implements Plugin<Project> {
                 }
             }
             packaging.doFirst {
-                project.gradle.addListener(reproduceListener)
+                project.gradle.addListener(packagingReproListener)
             }
             packaging.doLast {
-                project.gradle.removeListener(reproduceListener)
+                project.gradle.removeListener(packagingReproListener)
             }
-
             if (project.extensions.esvagrant.boxes.contains(box)) {
                 packagingTest.dependsOn(packaging)
             }
+
+            Task platform = project.tasks.create("vagrant${boxTask}#platformTest", VagrantCommandTask) {
+                boxName box
+                environmentVars vagrantEnvVars
+                dependsOn up
+                finalizedBy halt
+                args 'ssh', boxName, '--command', PLATFORM_TEST_COMMAND + " -Dtests.seed=${-> project.extensions.esvagrant.formattedTestSeed}"
+            }
+            TaskExecutionAdapter platformReproListener = new TaskExecutionAdapter() {
+                @Override
+                void afterExecute(Task task, TaskState state) {
+                    if (state.failure != null) {
+                        println "REPRODUCE WITH: gradle ${platform.path} " +
+                            "-Dtests.seed=${project.extensions.esvagrant.formattedTestSeed} "
+                    }
+                }
+            }
+            packaging.doFirst {
+                project.gradle.addListener(platformReproListener)
+            }
+            packaging.doLast {
+                project.gradle.removeListener(platformReproListener)
+            }
+            if (project.extensions.esvagrant.boxes.contains(box)) {
+                platformTest.dependsOn(platform)
+            }
         }
     }
 }