[Gradle] Add buildkite ci validation script for configuration cache (#110269)

We see gradle cache incompatibilities sneaking into our code base. Until
we can enable this feature by default we should regular run checks
existing configuration cache achievements are not degrading.

This change includes: - updating spotless gradle plugin to configuration
cache compatible version (its still BETA but imo good enough for our
needs)

.buildkite/scripts/gradle-cache-validation.sh → .buildkite/scripts/gradle-build-cache-validation.sh

@@ -12,17 +12,18 @@ curl -s -L -O https://github.com/gradle/gradle-enterprise-build-validation-scrip
 tmpOutputFile=$(mktemp)
 trap "rm $tmpOutputFile" EXIT
 
+set +e
 gradle-enterprise-gradle-build-validation/03-validate-local-build-caching-different-locations.sh -r https://github.com/elastic/elasticsearch.git -b $BUILDKITE_BRANCH --gradle-enterprise-server https://gradle-enterprise.elastic.co -t precommit --fail-if-not-fully-cacheable | tee $tmpOutputFile
-
 # Capture the return value
 retval=$?
+set -e
 
 # Now read the content from the temporary file into a variable
 perfOutput=$(cat $tmpOutputFile | sed -n '/Performance Characteristics/,/See https:\/\/gradle.com\/bvs\/main\/Gradle.md#performance-characteristics for details./p' | sed '$d' | sed 's/\x1b\[[0-9;]*m//g')
 investigationOutput=$(cat $tmpOutputFile | sed -n '/Investigation Quick Links/,$p' | sed 's/\x1b\[[0-9;]*m//g')
 
 # Initialize HTML output variable
-summaryHtml="<h4>Performance Characteristics</h4>"
+summaryHtml="<h4>Build Cache Performance Characteristics</h4>"
 summaryHtml+="<ul>"
 
 # Process each line of the string

.buildkite/scripts/gradle-configuration-cache-validation.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# TODO/ FIXIT without a full resolved gradle home, we see issues configuration cache reuse
+./gradlew --max-workers=8 --parallel --scan --no-daemon precommit
+
+./gradlew --max-workers=8 --parallel --scan --configuration-cache precommit -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2
+
+# Create a temporary file
+tmpOutputFile=$(mktemp)
+trap "rm $tmpOutputFile" EXIT
+
+echo "2nd run"
+# TODO run-gradle.sh script causes issues because of init script handling
+./gradlew --max-workers=8 --parallel --scan --configuration-cache precommit -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2 | tee $tmpOutputFile
+
+# Check if the command was successful
+if grep -q "Configuration cache entry reused." $tmpOutputFile; then
+    echo "Gradle configuration cache reused"
+    exit 0
+else
+    echo "Failed to reuse Gradle configuration cache."
+    exit 1
+fi
+
+

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java

@@ -61,7 +61,7 @@ public abstract class ElasticsearchBuildCompletePlugin implements Plugin<Project
             : System.getenv("BUILDKITE_BUILD_NUMBER");
         String performanceTest = System.getenv("BUILD_PERFORMANCE_TEST");
         if (buildNumber != null && performanceTest == null && GradleUtils.isIncludedBuild(target) == false) {
-            File targetFile = target.file("build/" + buildNumber + ".tar.bz2");
+            File targetFile = calculateTargetFile(target, buildNumber);
             File projectDir = target.getProjectDir();
             File gradleWorkersDir = new File(target.getGradle().getGradleUserHomeDir(), "workers/");
             DevelocityConfiguration extension = target.getExtensions().getByType(DevelocityConfiguration.class);
@@ -86,9 +86,19 @@ public abstract class ElasticsearchBuildCompletePlugin implements Plugin<Project
         }
     }
 
+    private File calculateTargetFile(Project target, String buildNumber) {
+        File uploadFile = target.file("build/" + buildNumber + ".tar.bz2");
+        int artifactIndex = 1;
+        while (uploadFile.exists()) {
+            uploadFile = target.file("build/" + buildNumber + "-" + artifactIndex++ + ".tar.bz2");
+        }
+        return uploadFile;
+    }
+
     private List<File> resolveProjectLogs(File projectDir) {
         var projectDirFiles = getFileOperations().fileTree(projectDir);
         projectDirFiles.include("**/*.hprof");
+        projectDirFiles.include("**/build/reports/configuration-cache/**");
         projectDirFiles.include("**/build/test-results/**/*.xml");
         projectDirFiles.include("**/build/testclusters/**");
         projectDirFiles.include("**/build/testrun/*/temp/**");

x-pack/plugin/esql/build.gradle

@@ -50,10 +50,12 @@ dependencies {
   internalClusterTestImplementation project(":modules:mapper-extras")
 }
 
+def projectDirectory = project.layout.projectDirectory
+def generatedSourceDir = projectDirectory.dir("src/main/generated")
 tasks.named("compileJava").configure {
-  options.compilerArgumentProviders.add(new SourceDirectoryCommandLineArgumentProvider(project.layout.projectDirectory.dir("src/main/generated")))
+  options.compilerArgumentProviders.add(new SourceDirectoryCommandLineArgumentProvider(generatedSourceDir))
   // IntelliJ sticks generated files here and we can't stop it....
-  exclude { it.file.toString().contains("$projectDir/src/main/generated-src/generated") }
+  exclude { it.file.toString().contains("src/main/generated-src/generated") }
 }
 
 interface Injected {

x-pack/plugin/esql/compute/build.gradle

@@ -18,8 +18,10 @@ dependencies {
   testImplementation(project(xpackModule('esql-core')))
 }
 
+def projectDirectory = project.layout.projectDirectory
+def generatedSourceDir = projectDirectory.dir("src/main/generated")
 tasks.named("compileJava").configure {
-  options.compilerArgumentProviders.add(new SourceDirectoryCommandLineArgumentProvider(project.layout.projectDirectory.dir("src/main/generated")))
+  options.compilerArgumentProviders.add(new SourceDirectoryCommandLineArgumentProvider(generatedSourceDir))
 }
 
 tasks.named('checkstyleMain').configure {