Refactor integration tests

1. tests don't have a bogus test dependency on zips anymore,
   instead we handle this in pre-integration-test. This reduces
   lots of confusion for e.g. mvn clean test.
2. refactor integ logic so that core/ and plugin/ share it.
   previously they were duplicates but the above change simplifies life.
   it also makes it easier for doing more interesting stuff

dev-tools/src/main/resources/ant/integration-tests.xml

@@ -16,7 +16,7 @@
   <property name="integ.args"
             value="-Des.node.name=smoke_tester -Des.cluster.name=prepare_release
                    -Des.discovery.zen.ping.multicast.enabled=false -Des.script.inline=on
-                   -Des.script.indexed=on -Des.pidfile=${integ.pidfile}"/>
+                   -Des.script.indexed=on -Des.pidfile=${integ.pidfile} -Des.repositories.url.allowed_urls=http://snapshot.test*"/>
 
   <!-- runs an OS script -->
   <macrodef name="run-script">
@@ -72,85 +72,77 @@
     </sequential>
   </macrodef>
 
-  <!-- unzip core release artifact and start ES -->
-  <target name="start-external-cluster" depends="stop-external-cluster" unless="${shouldskip}">
-    <local name="integ.finalname"/>
-    <local name="integ.home"/>
-    <local name="integ.repo.home"/>
-    <local name="integ.pid"/>
-
-    <delete dir="${integ.scratch}"/>
-    <property name="integ.finalname" value="${project.artifactId}-${project.version}"/>
+  <!-- installs a plugin into elasticsearch -->
+  <macrodef name="install-plugin">
+      <attribute name="home" default="${integ.scratch}/elasticsearch-${elasticsearch.version}"/>
+      <attribute name="name"/>
+      <attribute name="file"/>
+    <sequential>
+      <local name="url"/>
+      <makeurl property="url" file="@{file}"/>
+
+      <!-- install plugin -->
+      <echo>Installing plugin @{name}...</echo>
+      <run-script dir="@{home}" script="bin/plugin" args="-u ${url} -i @{name}"/>
+
+      <!-- check that plugin was installed into correct place -->
+      <local name="longname"/>
+      <property name="longname" value="@{name}"/>
+      <local name="shortname"/>
+      <filter-property src="longname" dest="shortname">
+        <chain>
+          <replaceregex pattern="^elasticsearch-" replace=""/>
+        </chain>
+      </filter-property>
+
+      <fail message="did not find plugin installed as ${shortname}">
+        <condition>
+          <not>
+            <resourceexists>
+              <file file="@{home}/plugins/${shortname}"/>
+            </resourceexists>
+          </not>
+        </condition>
+      </fail>
+    </sequential>
+  </macrodef>
 
-    <unzip src="${project.build.directory}/releases/${integ.finalname}.zip"
-           dest="${integ.scratch}"/>
+  <!-- start elasticsearch and wait until its ready -->
+  <macrodef name="startup-elasticsearch">
+      <attribute name="home" default="${integ.scratch}/elasticsearch-${elasticsearch.version}"/>
+      <attribute name="args" default="${integ.args}"/>
+    <sequential>
+      <echo>Starting up external cluster...</echo>
+      <run-script dir="@{home}" script="bin/elasticsearch" spawn="true"
+                  args="@{args} -Des.path.repo=@{home}/repo" />
 
-    <property name="integ.home" location="${integ.scratch}/${integ.finalname}"/>
-    <property name="integ.repo.home" location="${integ.home}/repo"/>
+      <waitfor maxwait="3" maxwaitunit="minute" checkevery="500">
+        <http url="http://127.0.0.1:9200"/>
+      </waitfor>
 
-    <!-- execute -->
-    <echo>Starting up external cluster...</echo>
-    <run-script dir="${integ.home}" script="bin/elasticsearch" spawn="true"
-                args="${integ.args} -Des.path.repo=${integ.repo.home} -Des.repositories.url.allowed_urls=http://snapshot.test*" />
+      <local name="integ.pid"/>
+      <extract-pid property="integ.pid"/>
+      <echo>External cluster started PID ${integ.pid}</echo>
+    </sequential>
+  </macrodef>
 
-    <waitfor maxwait="3" maxwaitunit="minute" checkevery="500">
-      <http url="http://127.0.0.1:9200"/>
-    </waitfor>
+  <!-- unzip the elasticsearch zip -->
+  <target name="setup-workspace" depends="stop-external-cluster">
+    <sequential>
+      <delete dir="${integ.scratch}"/>
+      <unzip src="${integ.deps}/elasticsearch-${elasticsearch.version}.zip" dest="${integ.scratch}"/>
+    </sequential>
+  </target>
 
-    <extract-pid property="integ.pid"/>
-    <echo>External cluster started PID ${integ.pid}</echo>
+  <!-- unzip core release artifact and start ES -->
+  <target name="start-external-cluster" depends="setup-workspace" unless="${shouldskip}">
+    <startup-elasticsearch/>
   </target>
 
   <!-- unzip core release artifact, install plugin, then start ES -->
-  <target name="start-external-cluster-with-plugin" depends="stop-external-cluster" unless="${shouldskip}">
-    <local name="integ.home"/>
-    <local name="integ.repo.home"/>
-    <local name="integ.plugin.url"/>
-    <local name="integ.pid"/>
-
-    <delete dir="${integ.scratch}"/>
-    <unzip src="${org.elasticsearch:elasticsearch:zip}"
-           dest="${integ.scratch}"/>
-
-    <property name="integ.home" location="${integ.scratch}/elasticsearch-${elasticsearch.version}"/>
-    <property name="integ.repo.home" location="${integ.home}/repo"/>
-
-    <makeurl property="integ.plugin.url" file="${project.build.directory}/releases/${project.artifactId}-${project.version}.zip"/>
-
-    <!-- install plugin -->
-    <echo>Installing plugin ${project.artifactId}...</echo>
-    <run-script dir="${integ.home}" script="bin/plugin"
-                args="-u ${integ.plugin.url} -i ${project.artifactId}"/>
-
-    <!-- check that plugin was installed into correct place -->
-    <local name="expected-shortname"/>
-    <filter-property src="project.artifactId" dest="expected-shortname">
-      <chain>
-        <replaceregex pattern="^elasticsearch-" replace=""/>
-      </chain>
-    </filter-property>
-
-    <fail message="did not find plugin installed as ${expected-shortname}">
-      <condition>
-        <not>
-          <resourceexists>
-            <file file="${integ.home}/plugins/${expected-shortname}"/>
-          </resourceexists>
-        </not>
-      </condition>
-    </fail>
-
-    <!-- execute -->
-    <echo>Starting up external cluster...</echo>
-    <run-script dir="${integ.home}" script="bin/elasticsearch" spawn="true"
-                args="${integ.args} -Des.path.repo=${integ.repo.home} -Des.repositories.url.allowed_urls=http://snapshot.test*"/>
-
-    <waitfor maxwait="3" maxwaitunit="minute" checkevery="500">
-      <http url="http://127.0.0.1:9200"/>
-    </waitfor>
-
-    <extract-pid property="integ.pid"/>
-    <echo>External cluster started PID ${integ.pid}</echo>
+  <target name="start-external-cluster-with-plugin" depends="setup-workspace" unless="${shouldskip}">
+    <install-plugin name="${project.artifactId}" file="${project.build.directory}/releases/${project.artifactId}-${project.version}.zip"/>
+    <startup-elasticsearch/>
   </target>
 
   <!-- TODO, for some more safety, add back some of the old jps logic

plugins/pom.xml

@@ -43,12 +43,6 @@
             <type>test-jar</type>
             <scope>test</scope>
         </dependency>
-        <dependency>
-            <groupId>org.elasticsearch</groupId>
-            <artifactId>elasticsearch</artifactId>
-            <type>zip</type>
-            <scope>test</scope>
-        </dependency>
 
         <!-- Provided dependencies by elasticsearch itself  -->
         <dependency>

pom.xml

@@ -105,6 +105,7 @@
         <skip.unit.tests>${skipTests}</skip.unit.tests>
         <skip.integ.tests>${skipTests}</skip.integ.tests>
         <integ.scratch>${project.build.directory}/integ-tests</integ.scratch>
+        <integ.deps>${project.build.directory}/integ-deps</integ.deps>
         <integ.temp>${integ.scratch}/temp</integ.temp>
         <no.commit.pattern>\bno(n|)commit\b</no.commit.pattern>
     </properties>
@@ -464,6 +465,10 @@
                     <groupId>org.apache.maven.plugins</groupId>
                     <artifactId>maven-compiler-plugin</artifactId>
                 </plugin>
+                <plugin>
+                    <groupId>org.apache.maven.plugins</groupId>
+                    <artifactId>maven-dependency-plugin</artifactId>
+                </plugin>
                 <plugin>
                     <groupId>org.codehaus.mojo</groupId>
                     <artifactId>buildnumber-maven-plugin</artifactId>
@@ -874,10 +879,30 @@
                     </configuration>
                 </plugin>
                 <plugin>
-                    <!-- We just declare which plugin version to use. Each project can have then its own settings -->
                     <groupId>org.apache.maven.plugins</groupId>
                     <artifactId>maven-dependency-plugin</artifactId>
                     <version>2.10</version>
+                    <executions>
+                        <execution>
+                            <id>integ-setup-dependencies</id>
+                            <phase>pre-integration-test</phase>
+                            <goals>
+                                <goal>copy</goal>
+                            </goals>
+                            <configuration>
+                                <artifactItems>
+                                    <artifactItem>
+                                        <groupId>org.elasticsearch</groupId>
+                                        <artifactId>elasticsearch</artifactId>
+                                        <version>${elasticsearch.version}</version>
+                                        <type>zip</type>
+                                        <overWrite>true</overWrite>
+                                    </artifactItem>
+                                </artifactItems>
+                                <outputDirectory>${integ.deps}</outputDirectory>
+                            </configuration>
+                        </execution>
+                    </executions>
                 </plugin>
                 <plugin>
                     <!-- We just declare which plugin version to use. Each project can have then its own settings -->