Fix build cache misses caused by embedded reaper jar (#45404)

buildSrc/build.gradle

@@ -77,27 +77,13 @@ sourceSets {
     minimumRuntime { }
 }
 
-configurations {
-  reaper
-}
-
 compileMinimumRuntimeJava {
     targetCompatibility = 8
     sourceCompatibility = 8
 }
 
-normalization {
-    runtimeClasspath {
-        // Ignore the embedded JAR as we track this separately below as a runtime dependency
-        ignore 'META-INF/*.jar'
-    }
-}
-
 jar {
     from sourceSets.minimumRuntime.output
-    into('META-INF') {
-        from configurations.reaper
-    }
 }
 
 javadoc {
@@ -135,7 +121,6 @@ dependencies {
   testCompile "junit:junit:${props.getProperty('junit')}"
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${props.getProperty('randomizedrunner')}"
   testCompile 'com.github.tomakehurst:wiremock-jre8-standalone:2.23.2'
-  reaper project('reaper')
   minimumRuntimeCompile "junit:junit:${props.getProperty('junit')}"
   minimumRuntimeCompile localGroovy()
   minimumRuntimeCompile gradleApi()
@@ -184,9 +169,11 @@ if (project != rootProject) {
 
   configurations {
     distribution
+    reaper
   }
 
   dependencies {
+    reaper project('reaper')
     distribution project(':distribution:archives:windows-zip')
     distribution project(':distribution:archives:oss-windows-zip')
     distribution project(':distribution:archives:darwin-tar')
@@ -198,6 +185,9 @@ if (project != rootProject) {
   // for external projects we want to remove the marker file indicating we are running the Elasticsearch project
   processResources {
       exclude 'buildSrc.marker'
+      into('META-INF') {
+          from configurations.reaper
+      }
   }
 
   String localDownloads = "${rootProject.buildDir}/local-downloads"

buildSrc/reaper/build.gradle

@@ -1,4 +1,5 @@
 jar {
+    archiveName = "${project.name}.jar"
     manifest {
         attributes 'Main-Class': 'org.elasticsearch.gradle.reaper.Reaper'
     }

buildSrc/src/main/java/org/elasticsearch/gradle/ReaperService.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.gradle;
 
+import org.elasticsearch.gradle.tool.ClasspathUtils;
 import org.gradle.api.GradleException;
 import org.gradle.api.logging.Logger;
 import org.gradle.internal.jvm.Jvm;
@@ -27,11 +28,16 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.UncheckedIOException;
+import java.net.URL;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 public class ReaperService {
 
+    private static final String REAPER_CLASS = "org/elasticsearch/gradle/reaper/Reaper.class";
+    private static final Pattern REAPER_JAR_PATH_PATTERN = Pattern.compile("file:(.*)!/" + REAPER_CLASS);
     private Logger logger;
     private Path buildDir;
     private Path inputDir;
@@ -103,13 +109,7 @@ public class ReaperService {
     private synchronized void ensureReaperStarted() {
         if (reaperProcess == null) {
             try {
-                // copy the reaper jar
-                Path jarPath = buildDir.resolve("reaper").resolve("reaper.jar");
-                Files.createDirectories(jarPath.getParent());
-                InputStream jarInput = ReaperPlugin.class.getResourceAsStream("/META-INF/reaper.jar");
-                try (OutputStream out = Files.newOutputStream(jarPath)) {
-                    jarInput.transferTo(out);
-                }
+                Path jarPath = locateReaperJar();
 
                 // ensure the input directory exists
                 Files.createDirectories(inputDir);
@@ -134,6 +134,41 @@ public class ReaperService {
         }
     }
 
+    private Path locateReaperJar() {
+        if (ClasspathUtils.isElasticsearchProject()) {
+            // when running inside the Elasticsearch build just pull find the jar in the runtime classpath
+            URL main = this.getClass().getClassLoader().getResource(REAPER_CLASS);
+            String mainPath = main.getFile();
+            Matcher matcher = REAPER_JAR_PATH_PATTERN.matcher(mainPath);
+
+            if (matcher.matches()) {
+                return Path.of(matcher.group(1));
+            } else {
+                throw new RuntimeException("Unable to locate " + REAPER_CLASS + " on build classpath.");
+            }
+        } else {
+            // copy the reaper jar
+            Path jarPath = buildDir.resolve("reaper").resolve("reaper.jar");
+            try {
+                Files.createDirectories(jarPath.getParent());
+            } catch (IOException e) {
+                throw new UncheckedIOException("Unable to create reaper JAR output directory " + jarPath.getParent(), e);
+            }
+
+            try (
+                OutputStream out = Files.newOutputStream(jarPath);
+                InputStream jarInput = this.getClass().getResourceAsStream("/META-INF/reaper.jar");
+            ) {
+                logger.info("Copying reaper.jar...");
+                jarInput.transferTo(out);
+            } catch (IOException e) {
+                throw new UncheckedIOException(e);
+            }
+
+            return jarPath;
+        }
+    }
+
     private void ensureReaperAlive() {
         if (reaperProcess.isAlive() == false) {
             throw new IllegalStateException("Reaper process died unexpectedly! Check the log at " + logFile.toString());

buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.reaper.properties

@@ -0,0 +1 @@
+implementation-class=org.elasticsearch.gradle.ReaperPlugin

buildSrc/src/test/java/org/elasticsearch/gradle/ReaperPluginIT.java

@@ -0,0 +1,21 @@
+package org.elasticsearch.gradle;
+
+import org.elasticsearch.gradle.test.GradleIntegrationTestCase;
+import org.gradle.testkit.runner.BuildResult;
+import org.gradle.testkit.runner.GradleRunner;
+import org.junit.Before;
+
+public class ReaperPluginIT extends GradleIntegrationTestCase {
+    private GradleRunner runner;
+
+    @Before
+    public void setup() {
+        runner = getGradleRunner("reaper");
+    }
+
+    public void testCanLaunchReaper() {
+        BuildResult result = runner.withArguments(":launchReaper", "-S", "--info").build();
+        assertTaskSuccessful(result, ":launchReaper");
+        assertOutputContains(result.getOutput(), "Copying reaper.jar...");
+    }
+}

buildSrc/src/testKit/reaper/build.gradle

@@ -0,0 +1,11 @@
+plugins {
+    id 'elasticsearch.reaper'
+}
+
+task launchReaper {
+    doLast {
+        def reaper = project.extensions.getByName('reaper')
+        reaper.registerCommand('test', 'true')
+        reaper.unregister('test')
+    }
+}