Make gradle build finished logic CC compatible (#96475)

* Make gradle build finished logic CC compatible
* Make ElasticsearchBuildFinishedPlugin configuration cache aware
* Add gradle enterprise plugin to buildlibs version catalogue

build-tools-internal/build.gradle

@@ -35,6 +35,10 @@ gradlePlugin {
       id = 'elasticsearch.build'
       implementationClass = 'org.elasticsearch.gradle.internal.BuildPlugin'
     }
+    buildFinished {
+      id = 'elasticsearch.build-finished'
+      implementationClass = 'org.elasticsearch.gradle.internal.ElasticsearchBuildFinishedPlugin'
+    }
     distro {
       id = 'elasticsearch.distro'
       implementationClass = 'org.elasticsearch.gradle.internal.distribution.ElasticsearchDistributionPlugin'
@@ -266,6 +270,8 @@ dependencies {
   api buildLibs.apache.rat
   api buildLibs.jna
   api buildLibs.shadow.plugin
+  api buildLibs.gradle.enterprise
+
   // for our ide tweaking
   api buildLibs.idea.ext
   // When upgrading forbidden apis, ensure dependency version is bumped in ThirdPartyPrecommitPlugin as well
@@ -280,6 +286,7 @@ dependencies {
   api buildLibs.asm.tree
   api buildLibs.httpclient
   api buildLibs.httpcore
+
   compileOnly buildLibs.checkstyle
   runtimeOnly "org.elasticsearch.gradle:reaper:$version"
   testImplementation buildLibs.checkstyle

build-tools-internal/src/main/groovy/elasticsearch.build-complete.gradle

@@ -1,93 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-import org.elasticsearch.gradle.util.GradleUtils
-
-import java.nio.file.Files
-
-String buildNumber = System.getenv('BUILD_NUMBER') ?: System.getenv('BUILDKITE_BUILD_NUMBER')
-String performanceTest = System.getenv('BUILD_PERFORMANCE_TEST')
-Boolean isNested = System.getProperty("scan.tag.NESTED") != null
-
-if (buildNumber && performanceTest == null && GradleUtils.isIncludedBuild(project) == false && isNested == false) {
-  def uploadFilePath = "build/${buildNumber}.tar.bz2"
-  File uploadFile = file(uploadFilePath)
-  project.gradle.buildFinished { result ->
-    println "build complete, generating: $uploadFile"
-    if (uploadFile.exists()) {
-      project.delete(uploadFile)
-    }
-
-    try {
-      ant.tar(destfile: uploadFile, compression: "bzip2", longfile: "gnu") {
-        fileset(dir: projectDir) {
-          Set<File> fileSet = fileTree(projectDir) {
-            include("**/*.hprof")
-            include("**/build/test-results/**/*.xml")
-            include("**/build/testclusters/**")
-            include("**/build/testrun/*/temp/**")
-            include("**/build/**/hs_err_pid*.log")
-            exclude("**/build/testclusters/**/data/**")
-            exclude("**/build/testclusters/**/distro/**")
-            exclude("**/build/testclusters/**/repo/**")
-            exclude("**/build/testclusters/**/extract/**")
-            exclude("**/build/testclusters/**/tmp/**")
-            exclude("**/build/testrun/*/temp/**/data/**")
-            exclude("**/build/testrun/*/temp/**/distro/**")
-            exclude("**/build/testrun/*/temp/**/repo/**")
-            exclude("**/build/testrun/*/temp/**/extract/**")
-            exclude("**/build/testrun/*/temp/**/tmp/**")
-          }
-            .files
-            .findAll { Files.isRegularFile(it.toPath()) }
-
-          if (fileSet.empty) {
-            // In cases where we don't match any workspace files, exclude everything
-            ant.exclude(name: "**/*")
-          } else {
-            fileSet.each {
-              ant.include(name: projectDir.toPath().relativize(it.toPath()))
-            }
-          }
-        }
-
-        fileset(dir: "${gradle.gradleUserHomeDir}/daemon/${gradle.gradleVersion}", followsymlinks: false) {
-          include(name: "**/daemon-${ProcessHandle.current().pid()}*.log")
-        }
-
-        fileset(dir: "${gradle.gradleUserHomeDir}/workers", followsymlinks: false)
-
-        fileset(dir: "${project.projectDir}/.gradle/reaper", followsymlinks: false, erroronmissingdir: false)
-      }
-    } catch (Exception e) {
-      logger.lifecycle("Failed to archive additional logs", e)
-    }
-
-    if (uploadFile.exists() && System.getenv("BUILDKITE") == "true") {
-      try {
-        println "Uploading buildkite artifact: ${uploadFilePath}..."
-        new ProcessBuilder("buildkite-agent", "artifact", "upload", uploadFilePath)
-          .start()
-          .waitFor()
-
-        println "Generating buildscan link for artifact..."
-
-        def process = new ProcessBuilder("buildkite-agent", "artifact", "search", uploadFilePath, "--step", System.getenv('BUILDKITE_JOB_ID'), "--format", "%i").start()
-        process.waitFor()
-        def artifactUuid = (process.text ?: "").trim()
-
-        println "Artifact UUID: ${artifactUuid}"
-        if (artifactUuid) {
-          buildScan.link 'Artifact Upload', "https://buildkite.com/organizations/elastic/pipelines/${System.getenv('BUILDKITE_PIPELINE_SLUG')}/builds/${buildNumber}/jobs/${System.getenv('BUILDKITE_JOB_ID')}/artifacts/${artifactUuid}"
-        }
-      } catch (Exception e) {
-        logger.lifecycle("Failed to upload buildkite artifact", e)
-      }
-    }
-  }
-}

build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle

@@ -132,6 +132,7 @@ buildScan {
       }
 
       buildFinished { result ->
+
         buildScanPublished { scan ->
           // Attach build scan link as build metadata
           // See: https://buildkite.com/docs/pipelines/build-meta-data

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildFinishedPlugin.java

@@ -0,0 +1,219 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal;
+
+import com.gradle.scan.plugin.BuildScanExtension;
+
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
+import org.apache.commons.io.IOUtils;
+import org.elasticsearch.gradle.util.GradleUtils;
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.file.FileSystemOperations;
+import org.gradle.api.flow.FlowAction;
+import org.gradle.api.flow.FlowParameters;
+import org.gradle.api.flow.FlowProviders;
+import org.gradle.api.flow.FlowScope;
+import org.gradle.api.internal.file.FileOperations;
+import org.gradle.api.provider.ListProperty;
+import org.gradle.api.provider.Property;
+import org.gradle.api.tasks.Input;
+import org.jetbrains.annotations.NotNull;
+
+import java.io.*;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import javax.inject.Inject;
+
+public abstract class ElasticsearchBuildFinishedPlugin implements Plugin<Project> {
+
+    @Inject
+    protected abstract FlowScope getFlowScope();
+
+    @Inject
+    protected abstract FlowProviders getFlowProviders();
+
+    @Inject
+    protected abstract FileOperations getFileOperations();
+
+    @Override
+    public void apply(Project target) {
+        String buildNumber = System.getenv("BUILD_NUMBER") != null
+            ? System.getenv("BUILD_NUMBER")
+            : System.getenv("BUILDKITE_BUILD_NUMBER");
+        String performanceTest = System.getenv("BUILD_PERFORMANCE_TEST");
+        if (buildNumber != null && performanceTest == null && GradleUtils.isIncludedBuild(target) == false) {
+            File targetFile = target.file("build/" + buildNumber + ".tar.bz2");
+            File projectDir = target.getProjectDir();
+            File gradleWorkersDir = new File(target.getGradle().getGradleUserHomeDir(), "workers/");
+            BuildScanExtension extension = target.getExtensions().getByType(BuildScanExtension.class);
+            File daemonsLogDir = new File(target.getGradle().getGradleUserHomeDir(), "daemon/" + target.getGradle().getGradleVersion());
+
+            getFlowScope().always(BuildFinishedFlowAction.class, spec -> {
+                spec.getParameters().getBuildScan().set(extension);
+                spec.getParameters().getUploadFile().set(targetFile);
+                spec.getParameters().getProjectDir().set(projectDir);
+                spec.getParameters().getFilteredFiles().addAll(getFlowProviders().getBuildWorkResult().map((result) -> {
+                    List<File> files = new ArrayList<>();
+                    files.addAll(resolveProjectLogs(projectDir));
+                    if (files.isEmpty() == false) {
+                        files.addAll(resolveDaemonLogs(daemonsLogDir));
+                        files.addAll(getFileOperations().fileTree(gradleWorkersDir).getFiles());
+                        files.addAll(getFileOperations().fileTree(new File(projectDir, ".gradle/reaper/")).getFiles());
+                    }
+                    return files;
+                }));
+            });
+        }
+    }
+
+    private List<File> resolveProjectLogs(File projectDir) {
+        var projectDirFiles = getFileOperations().fileTree(projectDir);
+        projectDirFiles.include("**/*.hprof");
+        projectDirFiles.include("**/build/test-results/**/*.xml");
+        projectDirFiles.include("**/build/testclusters/**");
+        projectDirFiles.include("**/build/testrun/*/temp/**");
+        projectDirFiles.include("**/build/**/hs_err_pid*.log");
+        projectDirFiles.exclude("**/build/testclusters/**/data/**");
+        projectDirFiles.exclude("**/build/testclusters/**/distro/**");
+        projectDirFiles.exclude("**/build/testclusters/**/repo/**");
+        projectDirFiles.exclude("**/build/testclusters/**/extract/**");
+        projectDirFiles.exclude("**/build/testclusters/**/tmp/**");
+        projectDirFiles.exclude("**/build/testrun/*/temp/**/data/**");
+        projectDirFiles.exclude("**/build/testrun/*/temp/**/distro/**");
+        projectDirFiles.exclude("**/build/testrun/*/temp/**/repo/**");
+        projectDirFiles.exclude("**/build/testrun/*/temp/**/extract/**");
+        projectDirFiles.exclude("**/build/testrun/*/temp/**/tmp/**");
+        return projectDirFiles.getFiles().stream().filter(f -> Files.isRegularFile(f.toPath())).toList();
+    }
+
+    private List<File> resolveDaemonLogs(File daemonsLogDir) {
+        var gradleDaemonFileSet = getFileOperations().fileTree(daemonsLogDir);
+        gradleDaemonFileSet.include("**/daemon-" + ProcessHandle.current().pid() + "*.log");
+        return gradleDaemonFileSet.getFiles().stream().filter(f -> Files.isRegularFile(f.toPath())).toList();
+    }
+
+    public abstract static class BuildFinishedFlowAction implements FlowAction<BuildFinishedFlowAction.Parameters> {
+        interface Parameters extends FlowParameters {
+            @Input
+            Property<File> getUploadFile();
+
+            @Input
+            Property<File> getProjectDir();
+
+            @Input
+            ListProperty<File> getFilteredFiles();
+
+            @Input
+            Property<BuildScanExtension> getBuildScan();
+
+        }
+
+        @Inject
+        protected abstract FileSystemOperations getFileSystemOperations();
+
+        @SuppressWarnings("checkstyle:DescendantToken")
+        @Override
+        public void execute(BuildFinishedFlowAction.Parameters parameters) throws FileNotFoundException {
+            File uploadFile = parameters.getUploadFile().get();
+            if (uploadFile.exists()) {
+                getFileSystemOperations().delete(spec -> spec.delete(uploadFile));
+            }
+            uploadFile.getParentFile().mkdirs();
+            createBuildArchiveTar(parameters.getFilteredFiles().get(), parameters.getProjectDir().get(), uploadFile);
+            if (uploadFile.exists() && System.getenv("BUILDKITE").equals("true")) {
+                String uploadFilePath = "build/" + uploadFile.getName();
+                try {
+                    System.out.println("Uploading buildkite artifact: " + uploadFilePath + "...");
+                    new ProcessBuilder("buildkite-agent", "artifact", "upload", uploadFilePath).start().waitFor();
+
+                    System.out.println("Generating buildscan link for artifact...");
+
+                    Process process = new ProcessBuilder(
+                        "buildkite-agent",
+                        "artifact",
+                        "search",
+                        uploadFilePath,
+                        "--step",
+                        System.getenv("BUILDKITE_JOB_ID"),
+                        "--format",
+                        "%i"
+                    ).start();
+                    process.waitFor();
+                    String processOutput;
+                    try {
+                        processOutput = IOUtils.toString(process.getInputStream());
+                    } catch (IOException e) {
+                        processOutput = "";
+                    }
+                    String artifactUuid = processOutput.trim();
+
+                    System.out.println("Artifact UUID: " + artifactUuid);
+                    if (artifactUuid.isEmpty() == false) {
+                        String buildkitePipelineSlug = System.getenv("BUILDKITE_PIPELINE_SLUG");
+                        String targetLink = "https://buildkite.com/organizations/elastic/pipelines/"
+                            + buildkitePipelineSlug
+                            + "/builds/"
+                            + System.getenv("BUILD_NUMBER")
+                            + "/jobs/"
+                            + System.getenv("BUILDKITE_JOB_ID")
+                            + "/artifacts/"
+                            + artifactUuid;
+                        parameters.getBuildScan().get().link("Artifact Upload", targetLink);
+                    }
+                } catch (Exception e) {
+                    System.out.println("Failed to upload buildkite artifact " + e.getMessage());
+                }
+            }
+
+        }
+
+        private static void createBuildArchiveTar(List<File> files, File projectDir, File uploadFile) {
+            try (
+                OutputStream fOut = Files.newOutputStream(uploadFile.toPath());
+                BufferedOutputStream buffOut = new BufferedOutputStream(fOut);
+                BZip2CompressorOutputStream bzOut = new BZip2CompressorOutputStream(buffOut);
+                TarArchiveOutputStream tOut = new TarArchiveOutputStream(bzOut)
+            ) {
+                Path projectPath = projectDir.toPath();
+                tOut.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX);
+                for (Path path : files.stream().map(File::toPath).collect(Collectors.toList())) {
+                    if (!Files.isRegularFile(path)) {
+                        throw new IOException("Support only file!");
+                    }
+
+                    TarArchiveEntry tarEntry = new TarArchiveEntry(path.toFile(), calculateArchivePath(path, projectPath));
+
+                    tOut.putArchiveEntry(tarEntry);
+
+                    // copy file to TarArchiveOutputStream
+                    Files.copy(path, tOut);
+                    tOut.closeArchiveEntry();
+
+                }
+                tOut.flush();
+                tOut.finish();
+
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+        @NotNull
+        private static String calculateArchivePath(Path path, Path projectPath) {
+            return path.startsWith(projectPath) ? projectPath.relativize(path).toString() : path.getFileName().toString();
+        }
+    }
+}

build.gradle

@@ -29,8 +29,8 @@ plugins {
   id 'lifecycle-base'
   id 'elasticsearch.docker-support'
   id 'elasticsearch.global-build-info'
+  id 'elasticsearch.build-finished'
   id 'elasticsearch.build-scan'
-  id 'elasticsearch.build-complete'
   id 'elasticsearch.jdk-download'
   id 'elasticsearch.internal-distribution-download'
   id 'elasticsearch.runtime-jdk-provision'

gradle/build.versions.toml

@@ -17,6 +17,7 @@ commons-codec = "commons-codec:commons-codec:1.11"
 commmons-io = "commons-io:commons-io:2.2"
 docker-compose = "com.avast.gradle:gradle-docker-compose-plugin:0.17.5"
 forbiddenApis = "de.thetaphi:forbiddenapis:3.6"
+gradle-enterprise = "com.gradle:gradle-enterprise-gradle-plugin:3.14.1"
 hamcrest = "org.hamcrest:hamcrest:2.1"
 httpcore = "org.apache.httpcomponents:httpcore:4.4.12"
 httpclient = "org.apache.httpcomponents:httpclient:4.5.10"