Move public build api into included build (#72861)

This moves the public build api and plugins into a separete included build called 'build-tools' 
and we removed the duplication of included buildSrc twice (2nd import as build-tools).

The elasticsearch internal build logic is kept in build-tools-internal as included build which allows us better handling of this project that its just being an buildSrc project (e.g. we can reference tasks directly from the root build etc.)

Convention logic applied to both projects will live in a new build-conventions project.

build-conventions/build.gradle

@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+plugins {
+    id 'java-gradle-plugin'
+    id 'java-test-fixtures'
+}
+
+group = "org.elasticsearch"
+
+def minRuntimeJava = JavaVersion.toVersion(file('../build-tools-internal/src/main/resources/minimumRuntimeVersion').text)
+targetCompatibility = minRuntimeJava
+sourceCompatibility = minRuntimeJava
+
+gradlePlugin {
+    // We already configure publication and we don't need or want the one that comes
+    // with the java-gradle-plugin
+    automatedPublishing = false
+    plugins {
+        internalLicenseheaders {
+            id = 'elasticsearch.internal-licenseheaders'
+            implementationClass = 'org.elasticsearch.gradle.internal.conventions.precommit.LicenseHeadersPrecommitPlugin'
+        }
+        publish {
+            id = 'elasticsearch.publish'
+            implementationClass = 'org.elasticsearch.gradle.internal.conventions.PublishPlugin'
+        }
+        licensing {
+            id = 'elasticsearch.licensing'
+            implementationClass = 'org.elasticsearch.gradle.internal.conventions.LicensingPlugin'
+        }
+        basics {
+            id = 'elasticsearch.basic-build-tool-conventions'
+            implementationClass = 'org.elasticsearch.gradle.internal.conventions.BasicBuildToolConventionsPlugin'
+        }
+    }
+}
+
+repositories {
+    mavenCentral()
+    gradlePluginPortal()
+}
+
+dependencies {
+    api 'org.apache.maven:maven-model:3.6.2'
+    api 'gradle.plugin.com.github.jengelman.gradle.plugins:shadow:7.0.0'
+    api 'org.apache.rat:apache-rat:0.11'
+}

build-conventions/settings.gradle

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+rootProject.name = 'build-conventions'

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/BasicBuildToolConventionsPlugin.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.conventions;
+
+import org.elasticsearch.gradle.internal.conventions.info.ParallelDetector;
+import org.elasticsearch.gradle.internal.conventions.util.Util;
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.tasks.bundling.Jar;
+import org.gradle.api.tasks.testing.Test;
+
+import java.io.File;
+
+public class BasicBuildToolConventionsPlugin implements Plugin<Project> {
+
+    @Override
+    public void apply(Project project) {
+        int defaultParallel = ParallelDetector.findDefaultParallel(project);
+        project.getTasks().withType(Test.class).configureEach(test -> {
+            test.onlyIf((t) -> Util.getBooleanProperty("tests.fips.enabled", false) == false);
+            test.setMaxParallelForks(defaultParallel);
+        });
+        // we put all our distributable files under distributions
+        project.getTasks().withType(Jar.class).configureEach(j ->
+                j.getDestinationDirectory().set(new File(project.getBuildDir(), "distributions"))
+        );
+    }
+}

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/GUtils.java

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.conventions;
+
+import java.util.Locale;
+
+public abstract class GUtils {
+
+    public static String capitalize(String s) {
+        return s.substring(0, 1).toUpperCase(Locale.ROOT) + s.substring(1);
+    }
+}

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/GitInfoPlugin.java

@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.conventions;
+
+import org.elasticsearch.gradle.internal.conventions.info.GitInfo;
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.model.ObjectFactory;
+import org.gradle.api.provider.Property;
+import org.gradle.api.provider.Provider;
+import org.gradle.api.provider.ProviderFactory;
+import org.gradle.initialization.layout.BuildLayout;
+
+import javax.inject.Inject;
+import java.io.File;
+
+class GitInfoPlugin implements Plugin<Project> {
+
+    private ProviderFactory factory;
+    private ObjectFactory objectFactory;
+
+    private Provider<String> revision;
+    private Property<GitInfo> gitInfo;
+
+    @Inject
+    public GitInfoPlugin(ProviderFactory factory, ObjectFactory objectFactory) {
+        this.factory = factory;
+        this.objectFactory = objectFactory;
+    }
+
+    @Override
+    public void apply(Project project) {
+        File rootDir = (project.getGradle().getParent() == null) ?
+                project.getRootDir() :
+                project.getGradle().getParent().getRootProject().getRootDir();
+
+        gitInfo = objectFactory.property(GitInfo.class).value(factory.provider(() ->
+            GitInfo.gitInfo(rootDir)
+        ));
+        gitInfo.disallowChanges();
+        gitInfo.finalizeValueOnRead();
+
+        revision = gitInfo.map(info -> info.getRevision() == null ? info.getRevision() : "master");
+    }
+
+    public Property<GitInfo> getGitInfo() {
+        return gitInfo;
+    }
+
+    public Provider<String> getRevision() {
+        return revision;
+    }
+}

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/LicensingPlugin.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.conventions;
+
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.provider.MapProperty;
+import org.gradle.api.provider.Property;
+import org.gradle.api.provider.Provider;
+import org.gradle.api.provider.ProviderFactory;
+
+import javax.inject.Inject;
+import java.util.Map;
+import java.util.concurrent.Callable;
+
+public class LicensingPlugin implements Plugin<Project> {
+    final static String ELASTIC_LICENSE_URL_PREFIX = "https://raw.githubusercontent.com/elastic/elasticsearch/";
+    final static String ELASTIC_LICENSE_URL_POSTFIX = "/licenses/ELASTIC-LICENSE-2.0.txt";
+
+    private ProviderFactory providerFactory;
+
+    @Inject
+    public LicensingPlugin(ProviderFactory providerFactory) {
+        this.providerFactory = providerFactory;
+    }
+
+    @Override
+    public void apply(Project project) {
+        Provider<String> revision = project.getRootProject().getPlugins().apply(GitInfoPlugin.class).getRevision();
+        Provider<String> licenseCommitProvider = providerFactory.provider(() ->
+             isSnapshotVersion(project) ? revision.get() : "v" + project.getVersion().toString()
+        );
+
+        MapProperty<String, String> licensesProperty = project.getObjects().mapProperty(String.class, String.class);
+        Provider<String> projectLicenseURL = licenseCommitProvider.map(licenseCommit -> ELASTIC_LICENSE_URL_PREFIX +
+                licenseCommit + ELASTIC_LICENSE_URL_POSTFIX);
+        // But stick the Elastic license url in project.ext so we can get it if we need to switch to it
+        project.getExtensions().getExtraProperties().set("elasticLicenseUrl", projectLicenseURL);
+
+        MapProperty<String, String> convention = licensesProperty.convention(
+                providerFactory.provider((Callable<Map<? extends String, ? extends String>>) () -> Map.of(
+                        "Server Side Public License, v 1", "https://www.mongodb.com/licensing/server-side-public-license",
+                        "Elastic License 2.0", projectLicenseURL.get())
+                )
+        );
+        // Default to the SSPL+Elastic dual license
+        project.getExtensions().getExtraProperties().set("licenseCommit", licenseCommitProvider);
+        project.getExtensions().getExtraProperties().set("projectLicenses", convention);
+    }
+
+    private boolean isSnapshotVersion(Project project) {
+        return project.getVersion().toString().endsWith("-SNAPSHOT");
+    }
+
+}

buildSrc/src/main/java/org/elasticsearch/gradle/internal/PublishPlugin.java → build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/PublishPlugin.java

@@ -6,22 +6,25 @@
  * Side Public License, v 1.
  */
 
-package org.elasticsearch.gradle.internal;
+package org.elasticsearch.gradle.internal.conventions;
 
+import org.elasticsearch.gradle.internal.conventions.precommit.PomValidationPrecommitPlugin;
 import com.github.jengelman.gradle.plugins.shadow.ShadowExtension;
 import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin;
 import groovy.util.Node;
-import org.elasticsearch.gradle.internal.info.BuildParams;
-import org.elasticsearch.gradle.internal.precommit.PomValidationPrecommitPlugin;
-import org.elasticsearch.gradle.internal.util.Util;
+import org.elasticsearch.gradle.internal.conventions.util.Util;
+import org.elasticsearch.gradle.internal.conventions.info.GitInfo;
 import org.gradle.api.NamedDomainObjectSet;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
-import org.gradle.api.Task;
 import org.gradle.api.XmlProvider;
 import org.gradle.api.plugins.BasePlugin;
 import org.gradle.api.plugins.BasePluginConvention;
+import org.gradle.api.plugins.JavaLibraryPlugin;
 import org.gradle.api.plugins.JavaPlugin;
+import org.gradle.api.provider.MapProperty;
+import org.gradle.api.provider.Property;
+import org.gradle.api.provider.Provider;
 import org.gradle.api.publish.PublishingExtension;
 import org.gradle.api.publish.maven.MavenPublication;
 import org.gradle.api.publish.maven.plugins.MavenPublishPlugin;
@@ -30,9 +33,10 @@ import org.gradle.api.tasks.SourceSet;
 import org.gradle.api.tasks.TaskProvider;
 import org.gradle.api.tasks.bundling.Jar;
 import org.gradle.language.base.plugins.LifecycleBasePlugin;
-import java.util.concurrent.Callable;
 
-import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure;
+import java.io.File;
+import java.util.Map;
+import java.util.concurrent.Callable;
 
 public class PublishPlugin implements Plugin<Project> {
 
@@ -41,15 +45,18 @@ public class PublishPlugin implements Plugin<Project> {
         project.getPluginManager().apply(BasePlugin.class);
         project.getPluginManager().apply(MavenPublishPlugin.class);
         project.getPluginManager().apply(PomValidationPrecommitPlugin.class);
-        configurePublications(project);
+        project.getPluginManager().apply(LicensingPlugin.class);
+
         configureJavadocJar(project);
         configureSourcesJar(project);
         configurePomGeneration(project);
+        configurePublications(project);
     }
 
     private void configurePublications(Project project) {
-        PublishingExtension publishingExtension = project.getExtensions().getByType(PublishingExtension.class);
-        MavenPublication publication = publishingExtension.getPublications().create("elastic", MavenPublication.class);
+        var publishingExtension = project.getExtensions().getByType(PublishingExtension.class);
+        var publication = publishingExtension.getPublications().create("elastic", MavenPublication.class);
+
         project.afterEvaluate(project1 -> {
             if (project1.getPlugins().hasPlugin(ShadowPlugin.class)) {
                 configureWithShadowPlugin(project1, publication);
@@ -57,6 +64,25 @@ public class PublishPlugin implements Plugin<Project> {
                 publication.from(project.getComponents().getByName("java"));
             }
         });
+        publication.getPom().withXml(xml -> {
+            var node = xml.asNode();
+            node.appendNode("inceptionYear", "2009");
+            var licensesNode = node.appendNode("licenses");
+            var projectLicenses = (MapProperty<String, String>) project.getExtensions().getExtraProperties().get("projectLicenses");
+            projectLicenses.get().entrySet().stream().sorted(Map.Entry.comparingByKey()).forEach(entry -> {
+                Node license = licensesNode.appendNode("license");
+                license.appendNode("name", entry.getKey());
+                license.appendNode("url", entry.getValue());
+                license.appendNode("distribution", "repo");
+            });
+            var developer = node.appendNode("developers").appendNode("developer");
+            developer.appendNode("name", "Elastic");
+            developer.appendNode("url", "https://www.elastic.co");
+        });
+        publishingExtension.getRepositories().maven(mavenArtifactRepository -> {
+            mavenArtifactRepository.setName("test");
+            mavenArtifactRepository.setUrl(new File(project.getRootProject().getBuildDir(), "local-test-repo"));
+        });
     }
 
     private static String getArchivesBaseName(Project project) {
@@ -67,27 +93,29 @@ public class PublishPlugin implements Plugin<Project> {
      * Configuration generation of maven poms.
      */
     private static void configurePomGeneration(Project project) {
-        TaskProvider<Task> generatePomTask = project.getTasks().register("generatePom");
+        Property<GitInfo> gitInfo = project.getRootProject().getPlugins().apply(GitInfoPlugin.class).getGitInfo();
+
+        var generatePomTask = project.getTasks().register("generatePom");
         project.getTasks().named(LifecycleBasePlugin.ASSEMBLE_TASK_NAME).configure(assemble -> assemble.dependsOn(generatePomTask));
         project.getTasks()
-            .withType(GenerateMavenPom.class)
-            .configureEach(
-                pomTask -> pomTask.setDestination(
-                    (Callable<String>) () -> String.format(
-                        "%s/distributions/%s-%s.pom",
-                        project.getBuildDir(),
-                        getArchivesBaseName(project),
-                        project.getVersion()
-                    )
-                )
-            );
-        PublishingExtension publishing = project.getExtensions().getByType(PublishingExtension.class);
+                .withType(GenerateMavenPom.class)
+                .configureEach(
+                        pomTask -> pomTask.setDestination(
+                                (Callable<String>) () -> String.format(
+                                        "%s/distributions/%s-%s.pom",
+                                        project.getBuildDir(),
+                                        getArchivesBaseName(project),
+                                        project.getVersion()
+                                )
+                        )
+                );
+        var publishing = project.getExtensions().getByType(PublishingExtension.class);
         final var mavenPublications = publishing.getPublications().withType(MavenPublication.class);
         addNameAndDescriptiontoPom(project, mavenPublications);
 
         mavenPublications.all(publication -> {
             // Add git origin info to generated POM files for internal builds
-            BuildParams.withInternalBuild(() -> publication.getPom().withXml(PublishPlugin::addScmInfo));
+            publication.getPom().withXml((xmlProvider) -> addScmInfo(xmlProvider, gitInfo.get()));
             // have to defer this until archivesBaseName is set
             project.afterEvaluate(p -> publication.setArtifactId(getArchivesBaseName(project)));
             generatePomTask.configure(t -> t.dependsOn(project.getTasks().withType(GenerateMavenPom.class)));
@@ -96,10 +124,10 @@ public class PublishPlugin implements Plugin<Project> {
 
     private static void addNameAndDescriptiontoPom(Project project, NamedDomainObjectSet<MavenPublication> mavenPublications) {
         mavenPublications.all(p -> p.getPom().withXml(xml -> {
-            Node root = xml.asNode();
+            var root = xml.asNode();
             root.appendNode("name", project.getName());
-            String description = project.getDescription() != null ? project.getDescription() : "";
-            root.appendNode("description", project.getDescription());
+            var description = project.getDescription() != null ? project.getDescription() : "";
+            root.appendNode("description", description);
         }));
     }
 
@@ -108,31 +136,31 @@ public class PublishPlugin implements Plugin<Project> {
         shadow.component(publication);
     }
 
-    private static void addScmInfo(XmlProvider xml) {
-        Node root = xml.asNode();
-        root.appendNode("url", Util.urlFromOrigin(BuildParams.getGitOrigin()));
-        Node scmNode = root.appendNode("scm");
-        scmNode.appendNode("url", BuildParams.getGitOrigin());
+    private static void addScmInfo(XmlProvider xml, GitInfo gitInfo) {
+        var root = xml.asNode();
+        root.appendNode("url", gitInfo.urlFromOrigin());
+        var scmNode = root.appendNode("scm");
+        scmNode.appendNode("url", gitInfo.getOrigin());
     }
 
     /**
      * Adds a javadocJar task to generate a jar containing javadocs.
      */
     private static void configureJavadocJar(Project project) {
-        project.getPlugins().withId("elasticsearch.java", p -> {
-            TaskProvider<Jar> javadocJarTask = project.getTasks().register("javadocJar", Jar.class);
+        project.getPlugins().withType(JavaLibraryPlugin.class, p -> {
+            var javadocJarTask = project.getTasks().register("javadocJar", Jar.class);
             javadocJarTask.configure(jar -> {
                 jar.getArchiveClassifier().set("javadoc");
                 jar.setGroup("build");
                 jar.setDescription("Assembles a jar containing javadocs.");
                 jar.from(project.getTasks().named(JavaPlugin.JAVADOC_TASK_NAME));
             });
-            maybeConfigure(project.getTasks(), BasePlugin.ASSEMBLE_TASK_NAME, t -> t.dependsOn(javadocJarTask));
+            project.getTasks().named(BasePlugin.ASSEMBLE_TASK_NAME).configure(t -> t.dependsOn(javadocJarTask));
         });
     }
 
     static void configureSourcesJar(Project project) {
-        project.getPlugins().withId("elasticsearch.java", p -> {
+        project.getPlugins().withType(JavaLibraryPlugin.class, p -> {
             TaskProvider<Jar> sourcesJarTask = project.getTasks().register("sourcesJar", Jar.class);
             sourcesJarTask.configure(jar -> {
                 jar.getArchiveClassifier().set("sources");
@@ -141,7 +169,7 @@ public class PublishPlugin implements Plugin<Project> {
                 SourceSet mainSourceSet = Util.getJavaMainSourceSet(project).get();
                 jar.from(mainSourceSet.getAllSource());
             });
-            maybeConfigure(project.getTasks(), BasePlugin.ASSEMBLE_TASK_NAME, t -> t.dependsOn(sourcesJarTask));
+            project.getTasks().named(BasePlugin.ASSEMBLE_TASK_NAME).configure(t -> t.dependsOn(sourcesJarTask));
         });
     }
 }

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/VersionPropertiesLoader.java

@@ -0,0 +1,65 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.conventions;
+
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Properties;
+
+// Define this here because we need it early.
+public class VersionPropertiesLoader {
+    static Properties loadBuildSrcVersion(File input) throws IOException {
+        Properties props = new Properties();
+        InputStream is = new FileInputStream(input);
+        try {
+            props.load(is);
+        } finally {
+            is.close();
+        }
+        loadBuildSrcVersion(props, System.getProperties());
+        return props;
+    }
+
+    protected static void loadBuildSrcVersion(Properties loadedProps, Properties systemProperties) {
+        String elasticsearch = loadedProps.getProperty("elasticsearch");
+        if (elasticsearch == null) {
+            throw new IllegalStateException("Elasticsearch version is missing from properties.");
+        }
+        if (elasticsearch.matches("[0-9]+\\.[0-9]+\\.[0-9]+") == false) {
+            throw new IllegalStateException(
+                    "Expected elasticsearch version to be numbers only of the form  X.Y.Z but it was: " +
+                            elasticsearch
+            );
+        }
+        String qualifier = systemProperties.getProperty("build.version_qualifier", "");
+        if (qualifier.isEmpty() == false) {
+            if (qualifier.matches("(alpha|beta|rc)\\d+") == false) {
+                throw new IllegalStateException("Invalid qualifier: " + qualifier);
+            }
+            elasticsearch += "-" + qualifier;
+        }
+        final String buildSnapshotSystemProperty = systemProperties.getProperty("build.snapshot", "true");
+        switch (buildSnapshotSystemProperty) {
+            case "true":
+                elasticsearch += "-SNAPSHOT";
+                break;
+            case "false":
+                // do nothing
+                break;
+            default:
+                throw new IllegalArgumentException(
+                        "build.snapshot was set to [" + buildSnapshotSystemProperty + "] but can only be unset or [true|false]");
+        }
+        loadedProps.put("elasticsearch", elasticsearch);
+    }
+}
+

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/info/GitInfo.java

@@ -0,0 +1,186 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.conventions.info;
+
+import org.gradle.api.GradleException;
+import org.gradle.api.logging.Logging;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class GitInfo {
+    private static final Pattern GIT_PATTERN = Pattern.compile("git@([^:]+):([^\\.]+)\\.git");
+
+    private final String revision;
+    private final String origin;
+
+    private GitInfo(String revision, String origin) {
+        this.revision = revision;
+        this.origin = origin;
+    }
+
+    public String getRevision() {
+        return revision;
+    }
+
+    public String getOrigin() {
+        return origin;
+    }
+
+    public static GitInfo gitInfo(File rootDir) {
+        try {
+            /*
+             * We want to avoid forking another process to run git rev-parse HEAD. Instead, we will read the refs manually. The
+             * documentation for this follows from https://git-scm.com/docs/gitrepository-layout and https://git-scm.com/docs/git-worktree.
+             *
+             * There are two cases to consider:
+             *  - a plain repository with .git directory at the root of the working tree
+             *  - a worktree with a plain text .git file at the root of the working tree
+             *
+             * In each case, our goal is to parse the HEAD file to get either a ref or a bare revision (in the case of being in detached
+             * HEAD state).
+             *
+             * In the case of a plain repository, we can read the HEAD file directly, resolved directly from the .git directory.
+             *
+             * In the case of a worktree, we read the gitdir from the plain text .git file. This resolves to a directory from which we read
+             * the HEAD file and resolve commondir to the plain git repository.
+             */
+            final Path dotGit = rootDir.toPath().resolve(".git");
+            final String revision;
+            if (Files.exists(dotGit) == false) {
+                return new GitInfo("unknown", "unknown");
+            }
+            final Path head;
+            final Path gitDir;
+            if (Files.isDirectory(dotGit)) {
+                // this is a git repository, we can read HEAD directly
+                head = dotGit.resolve("HEAD");
+                gitDir = dotGit;
+            } else {
+                // this is a git worktree, follow the pointer to the repository
+                final Path workTree = Paths.get(readFirstLine(dotGit).substring("gitdir:".length()).trim());
+                if (Files.exists(workTree) == false) {
+                    return new GitInfo("unknown", "unknown");
+                }
+                head = workTree.resolve("HEAD");
+                final Path commonDir = Paths.get(readFirstLine(workTree.resolve("commondir")));
+                if (commonDir.isAbsolute()) {
+                    gitDir = commonDir;
+                } else {
+                    // this is the common case
+                    gitDir = workTree.resolve(commonDir);
+                }
+            }
+            final String ref = readFirstLine(head);
+            if (ref.startsWith("ref:")) {
+                String refName = ref.substring("ref:".length()).trim();
+                Path refFile = gitDir.resolve(refName);
+                if (Files.exists(refFile)) {
+                    revision = readFirstLine(refFile);
+                } else if (Files.exists(gitDir.resolve("packed-refs"))) {
+                    // Check packed references for commit ID
+                    Pattern p = Pattern.compile("^([a-f0-9]{40}) " + refName + "$");
+                    try (Stream<String> lines = Files.lines(gitDir.resolve("packed-refs"))) {
+                        revision = lines.map(p::matcher)
+                                .filter(Matcher::matches)
+                                .map(m -> m.group(1))
+                                .findFirst()
+                                .orElseThrow(() -> new IOException("Packed reference not found for refName " + refName));
+                    }
+                } else {
+                    File refsDir = gitDir.resolve("refs").toFile();
+                    if (refsDir.exists()) {
+                        String foundRefs = Arrays.stream(refsDir.listFiles()).map(f -> f.getName()).collect(Collectors.joining("\n"));
+                        Logging.getLogger(GitInfo.class).error("Found git refs\n" + foundRefs);
+                    } else {
+                        Logging.getLogger(GitInfo.class).error("No git refs dir found");
+                    }
+                    throw new GradleException("Can't find revision for refName " + refName);
+                }
+            } else {
+                // we are in detached HEAD state
+                revision = ref;
+            }
+            return new GitInfo(revision, findOriginUrl(gitDir.resolve("config")));
+        } catch (final IOException e) {
+            // for now, do not be lenient until we have better understanding of real-world scenarios where this happens
+            throw new GradleException("unable to read the git revision", e);
+        }
+    }
+
+
+    private static String findOriginUrl(final Path configFile) throws IOException {
+        Map<String, String> props = new HashMap<>();
+
+        try (Stream<String> stream = Files.lines(configFile, StandardCharsets.UTF_8)) {
+            Iterator<String> lines = stream.iterator();
+            boolean foundOrigin = false;
+            while (lines.hasNext()) {
+                String line = lines.next().trim();
+                if (line.startsWith(";") || line.startsWith("#")) {
+                    // ignore comments
+                    continue;
+                }
+                if (foundOrigin) {
+                    if (line.startsWith("[")) {
+                        // we're on to the next config item so stop looking
+                        break;
+                    }
+                    String[] pair = line.trim().split("=", 2);
+                    props.put(pair[0].trim(), pair[1].trim());
+                } else {
+                    if (line.equals("[remote \"origin\"]")) {
+                        foundOrigin = true;
+                    }
+                }
+            }
+        }
+
+        String originUrl = props.get("url");
+        return originUrl == null ? "unknown" : originUrl;
+    }
+
+    private static String readFirstLine(final Path path) throws IOException {
+        String firstLine;
+        try (Stream<String> lines = Files.lines(path, StandardCharsets.UTF_8)) {
+            firstLine = lines.findFirst().orElseThrow(() -> new IOException("file [" + path + "] is empty"));
+        }
+        return firstLine;
+    }
+
+    /** Find the reponame. */
+    public String urlFromOrigin() {
+        String oritin = getOrigin();
+        if (origin == null) {
+            return null; // best effort, the url doesnt really matter, it is just required by maven central
+        }
+        if (origin.startsWith("https")) {
+            return origin;
+        }
+        Matcher matcher = GIT_PATTERN.matcher(origin);
+        if (matcher.matches()) {
+            return String.format("https://%s/%s", matcher.group(1), matcher.group(2));
+        } else {
+            return origin; // best effort, the url doesnt really matter, it is just required by maven central
+        }
+    }
+
+}

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/info/ParallelDetector.java

@@ -0,0 +1,83 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.conventions.info;
+
+import org.gradle.api.Project;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+public class ParallelDetector {
+
+    private static Integer _defaultParallel = null;
+
+    public static int findDefaultParallel(Project project) {
+        // Since it costs IO to compute this, and is done at configuration time we want to cache this if possible
+        // It's safe to store this in a static variable since it's just a primitive so leaking memory isn't an issue
+        if (_defaultParallel == null) {
+            File cpuInfoFile = new File("/proc/cpuinfo");
+            if (cpuInfoFile.exists()) {
+                // Count physical cores on any Linux distro ( don't count hyper-threading )
+                Map<String, Integer> socketToCore = new HashMap<>();
+                String currentID = "";
+
+                try (BufferedReader reader = new BufferedReader(new FileReader(cpuInfoFile))) {
+                    for (String line = reader.readLine(); line != null; line = reader.readLine()) {
+                        if (line.contains(":")) {
+                            List<String> parts = Arrays.stream(line.split(":", 2)).map(String::trim).collect(Collectors.toList());
+                            String name = parts.get(0);
+                            String value = parts.get(1);
+                            // the ID of the CPU socket
+                            if (name.equals("physical id")) {
+                                currentID = value;
+                            }
+                            // Number of cores not including hyper-threading
+                            if (name.equals("cpu cores")) {
+                                assert currentID.isEmpty() == false;
+                                socketToCore.put("currentID", Integer.valueOf(value));
+                                currentID = "";
+                            }
+                        }
+                    }
+                } catch (IOException e) {
+                    throw new UncheckedIOException(e);
+                }
+                _defaultParallel = socketToCore.values().stream().mapToInt(i -> i).sum();
+            } else if (isMac()) {
+                // Ask macOS to count physical CPUs for us
+                ByteArrayOutputStream stdout = new ByteArrayOutputStream();
+                project.exec(spec -> {
+                    spec.setExecutable("sysctl");
+                    spec.args("-n", "hw.physicalcpu");
+                    spec.setStandardOutput(stdout);
+                });
+
+                _defaultParallel = Integer.parseInt(stdout.toString().trim());
+            }
+
+            _defaultParallel = Runtime.getRuntime().availableProcessors() / 2;
+        }
+
+        return _defaultParallel;
+    }
+
+    private static boolean isMac() {
+        return System.getProperty("os.name", "").startsWith("Mac");
+    }
+
+}

buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/LicenseHeadersPrecommitPlugin.java → build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/LicenseHeadersPrecommitPlugin.java

@@ -6,14 +6,12 @@
  * Side Public License, v 1.
  */
 
-package org.elasticsearch.gradle.internal.precommit;
+package org.elasticsearch.gradle.internal.conventions.precommit;
 
-import org.elasticsearch.gradle.internal.InternalPlugin;
-import org.elasticsearch.gradle.precommit.PrecommitPlugin;
-import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.Project;
 import org.gradle.api.Task;
 import org.gradle.api.plugins.JavaBasePlugin;
+import org.gradle.api.plugins.JavaPluginConvention;
 import org.gradle.api.provider.ProviderFactory;
 import org.gradle.api.tasks.SourceSetContainer;
 import org.gradle.api.tasks.TaskProvider;
@@ -21,7 +19,7 @@ import org.gradle.api.tasks.TaskProvider;
 import javax.inject.Inject;
 import java.util.stream.Collectors;
 
-public class LicenseHeadersPrecommitPlugin extends PrecommitPlugin implements InternalPlugin {
+public class LicenseHeadersPrecommitPlugin extends PrecommitPlugin {
     @Inject
     public LicenseHeadersPrecommitPlugin(ProviderFactory providerFactory) {
         this.providerFactory = providerFactory;
@@ -31,7 +29,7 @@ public class LicenseHeadersPrecommitPlugin extends PrecommitPlugin implements In
     public TaskProvider<? extends Task> createTask(Project project) {
         return project.getTasks().register("licenseHeaders", LicenseHeadersTask.class, licenseHeadersTask -> {
             project.getPlugins().withType(JavaBasePlugin.class, javaBasePlugin -> {
-                final SourceSetContainer sourceSets = GradleUtils.getJavaSourceSets(project);
+                final SourceSetContainer sourceSets = project.getConvention().getPlugin(JavaPluginConvention.class).getSourceSets();
                 licenseHeadersTask.getSourceFolders()
                     .addAll(providerFactory.provider(() -> sourceSets.stream().map(s -> s.getAllJava()).collect(Collectors.toList())));
             });

buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/LicenseHeadersTask.java → build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/LicenseHeadersTask.java

@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-package org.elasticsearch.gradle.internal.precommit;
+package org.elasticsearch.gradle.internal.conventions.precommit;
 
 import org.apache.rat.Defaults;
 import org.apache.rat.ReportConfiguration;

buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/PomValidationPrecommitPlugin.java → build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/PomValidationPrecommitPlugin.java

@@ -6,10 +6,9 @@
  * Side Public License, v 1.
  */
 
-package org.elasticsearch.gradle.internal.precommit;
+package org.elasticsearch.gradle.internal.conventions.precommit;
 
-import org.elasticsearch.gradle.precommit.PrecommitPlugin;
-import org.elasticsearch.gradle.internal.util.Util;
+import org.elasticsearch.gradle.internal.conventions.GUtils;
 import org.gradle.api.Project;
 import org.gradle.api.Task;
 import org.gradle.api.publish.PublishingExtension;
@@ -26,7 +25,7 @@ public class PomValidationPrecommitPlugin extends PrecommitPlugin {
         TaskProvider<Task> validatePom = project.getTasks().register("validatePom");
         PublishingExtension publishing = project.getExtensions().getByType(PublishingExtension.class);
         publishing.getPublications().all(publication -> {
-            String publicationName = Util.capitalize(publication.getName());
+            String publicationName = GUtils.capitalize(publication.getName());
             TaskProvider<PomValidationTask> validateTask = project.getTasks()
                 .register("validate" + publicationName + "Pom", PomValidationTask.class);
             validatePom.configure(t -> t.dependsOn(validateTask));

buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/PomValidationTask.java → build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/PomValidationTask.java

@@ -6,11 +6,11 @@
  * Side Public License, v 1.
  */
 
-package org.elasticsearch.gradle.internal.precommit;
+package org.elasticsearch.gradle.internal.conventions.precommit;
 
 import org.apache.maven.model.Model;
 import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
-import org.elasticsearch.gradle.precommit.PrecommitTask;
+import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitTask;
 import org.gradle.api.GradleException;
 import org.gradle.api.file.RegularFileProperty;
 import org.gradle.api.tasks.InputFile;

buildSrc/src/main/java/org/elasticsearch/gradle/precommit/PrecommitPlugin.java → build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/PrecommitPlugin.java

@@ -6,12 +6,12 @@
  * Side Public License, v 1.
  */
 
-package org.elasticsearch.gradle.precommit;
+package org.elasticsearch.gradle.internal.conventions.precommit;
 
-import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.Task;
+import org.gradle.api.plugins.JavaPluginConvention;
 import org.gradle.api.tasks.TaskProvider;
 
 /**
@@ -27,10 +27,11 @@ public abstract class PrecommitPlugin implements Plugin<Project> {
         TaskProvider<? extends Task> task = createTask(project);
         TaskProvider<Task> precommit = project.getTasks().named(PRECOMMIT_TASK_NAME);
         precommit.configure(t -> t.dependsOn(task));
-
         project.getPluginManager().withPlugin("java", p -> {
             // We want to get any compilation error before running the pre-commit checks.
-            GradleUtils.getJavaSourceSets(project).all(sourceSet -> task.configure(t -> t.shouldRunAfter(sourceSet.getClassesTaskName())));
+            project.getConvention().getPlugin(JavaPluginConvention.class).getSourceSets().all(sourceSet ->
+                    task.configure(t -> t.shouldRunAfter(sourceSet.getClassesTaskName()))
+            );
         });
     }
 

buildSrc/src/main/java/org/elasticsearch/gradle/precommit/PrecommitTask.java → build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/PrecommitTask.java

@@ -5,7 +5,7 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-package org.elasticsearch.gradle.precommit;
+package org.elasticsearch.gradle.internal.conventions.precommit;
 
 import org.gradle.api.DefaultTask;
 import org.gradle.api.file.ProjectLayout;

buildSrc/src/main/java/org/elasticsearch/gradle/precommit/PrecommitTaskPlugin.java → build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/PrecommitTaskPlugin.java

@@ -6,13 +6,13 @@
  * Side Public License, v 1.
  */
 
-package org.elasticsearch.gradle.precommit;
+package org.elasticsearch.gradle.internal.conventions.precommit;
 
-import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.Task;
 import org.gradle.api.plugins.JavaBasePlugin;
+import org.gradle.api.plugins.JavaPluginConvention;
 import org.gradle.api.tasks.TaskProvider;
 import org.gradle.api.tasks.testing.Test;
 import org.gradle.language.base.plugins.LifecycleBasePlugin;
@@ -27,14 +27,15 @@ public class PrecommitTaskPlugin implements Plugin<Project> {
         });
 
         project.getPluginManager()
-            .withPlugin(
-                "lifecycle-base",
-                p -> project.getTasks().named(LifecycleBasePlugin.CHECK_TASK_NAME).configure(t -> t.dependsOn(precommit))
-            );
+                .withPlugin(
+                        "lifecycle-base",
+                        p -> project.getTasks().named(LifecycleBasePlugin.CHECK_TASK_NAME).configure(t -> t.dependsOn(precommit))
+                );
         project.getPluginManager().withPlugin("java", p -> {
             // run compilation as part of precommit
-            GradleUtils.getJavaSourceSets(project).all(sourceSet -> precommit.configure(t -> t.dependsOn(sourceSet.getClassesTaskName())));
-
+            project.getConvention().getPlugin(JavaPluginConvention.class).getSourceSets().all(sourceSet ->
+                    precommit.configure(t -> t.shouldRunAfter(sourceSet.getClassesTaskName()))
+            );
             // make sure tests run after all precommit tasks
             project.getTasks().withType(Test.class).configureEach(t -> t.mustRunAfter(precommit));
         });

buildSrc/src/main/java/org/elasticsearch/gradle/internal/util/Util.java → build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/util/Util.java

@@ -6,30 +6,22 @@
  * Side Public License, v 1.
  */
 
-package org.elasticsearch.gradle.internal.util;
+package org.elasticsearch.gradle.internal.conventions.util;
 
-import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin;
-import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.Action;
 import org.gradle.api.GradleException;
 import org.gradle.api.Project;
 import org.gradle.api.file.FileTree;
 import org.gradle.api.plugins.JavaPluginConvention;
 import org.gradle.api.tasks.SourceSet;
+import org.gradle.api.tasks.SourceSetContainer;
 import org.gradle.api.tasks.util.PatternFilterable;
 
 import javax.annotation.Nullable;
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.UncheckedIOException;
 import java.net.URI;
 import java.net.URISyntaxException;
-import java.util.Locale;
 import java.util.Optional;
 import java.util.function.Supplier;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 
 public class Util {
 
@@ -47,36 +39,6 @@ public class Util {
         }
     }
 
-    public static String getResourceContents(String resourcePath) {
-        try (
-            BufferedReader reader = new BufferedReader(new InputStreamReader(GlobalBuildInfoPlugin.class.getResourceAsStream(resourcePath)))
-        ) {
-            StringBuilder b = new StringBuilder();
-            for (String line = reader.readLine(); line != null; line = reader.readLine()) {
-                if (b.length() != 0) {
-                    b.append('\n');
-                }
-                b.append(line);
-            }
-
-            return b.toString();
-        } catch (IOException e) {
-            throw new UncheckedIOException("Error trying to read classpath resource: " + resourcePath, e);
-        }
-    }
-
-    public static String capitalize(String s) {
-        return s.substring(0, 1).toUpperCase(Locale.ROOT) + s.substring(1);
-    }
-
-    public static URI getBuildSrcCodeSource() {
-        try {
-            return Util.class.getProtectionDomain().getCodeSource().getLocation().toURI();
-        } catch (URISyntaxException e) {
-            throw new GradleException("Error determining build tools JAR location", e);
-        }
-    }
-
     /**
      * @param project The project to look for resources.
      * @param filter  Optional filter function to filter the returned resources
@@ -125,7 +87,7 @@ public class Util {
     public static Optional<SourceSet> getJavaTestSourceSet(Project project) {
         return project.getConvention().findPlugin(JavaPluginConvention.class) == null
             ? Optional.empty()
-            : Optional.ofNullable(GradleUtils.getJavaSourceSets(project).findByName(SourceSet.TEST_SOURCE_SET_NAME));
+            : Optional.ofNullable(getJavaSourceSets(project).findByName(SourceSet.TEST_SOURCE_SET_NAME));
     }
 
     /**
@@ -135,26 +97,9 @@ public class Util {
     public static Optional<SourceSet> getJavaMainSourceSet(Project project) {
         return project.getConvention().findPlugin(JavaPluginConvention.class) == null
             ? Optional.empty()
-            : Optional.ofNullable(GradleUtils.getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME));
+            : Optional.ofNullable(getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME));
     }
 
-    static final Pattern GIT_PATTERN = Pattern.compile("git@([^:]+):([^\\.]+)\\.git");
-
-    /** Find the reponame. */
-    public static String urlFromOrigin(String origin) {
-        if (origin == null) {
-            return null; // best effort, the url doesnt really matter, it is just required by maven central
-        }
-        if (origin.startsWith("https")) {
-            return origin;
-        }
-        Matcher matcher = GIT_PATTERN.matcher(origin);
-        if (matcher.matches()) {
-            return String.format("https://%s/%s", matcher.group(1), matcher.group(2));
-        } else {
-            return origin; // best effort, the url doesnt really matter, it is just required by maven central
-        }
-    }
 
     public static Object toStringable(Supplier<String> getter) {
         return new Object() {
@@ -164,4 +109,9 @@ public class Util {
             }
         };
     }
+
+    public static SourceSetContainer getJavaSourceSets(Project project) {
+        return project.getConvention().getPlugin(JavaPluginConvention.class).getSourceSets();
+    }
+
 }

buildSrc/build.gradle → build-tools-internal/build.gradle

@@ -6,26 +6,19 @@
  * Side Public License, v 1.
  */
 
-import org.gradle.internal.jvm.Jvm
+
+import org.elasticsearch.gradle.internal.conventions.VersionPropertiesLoader
 
 plugins {
   id 'java-gradle-plugin'
+  id 'groovy-gradle-plugin'
   id 'groovy'
-  id 'java-test-fixtures'
+  id 'elasticsearch.internal-licenseheaders'
+  id 'elasticsearch.basic-build-tool-conventions'
 }
 
 group = 'org.elasticsearch.gradle'
 
-if (project == rootProject) {
-  // change the build dir used during build init, so that doing a clean
-  // won't wipe out the buildscript jar
-  buildDir = 'build-bootstrap'
-}
-
-/*****************************************************************************
- *         Propagating version.properties to the rest of the build           *
- *****************************************************************************/
-
 // we update the version property to reflect if we are building a snapshot or a release build
 // we write this back out below to load it in the Build.java which will be shown in rest main action
 // to indicate this being a snapshot build or a release build.
@@ -41,10 +34,6 @@ gradlePlugin {
       id = 'elasticsearch.build'
       implementationClass = 'org.elasticsearch.gradle.internal.BuildPlugin'
     }
-    distributionDownload {
-      id = 'elasticsearch.distribution-download'
-      implementationClass = 'org.elasticsearch.gradle.DistributionDownloadPlugin'
-    }
     distroTest {
       id = 'elasticsearch.distro-test'
       implementationClass = 'org.elasticsearch.gradle.internal.test.DistroTestPlugin'
@@ -57,10 +46,6 @@ gradlePlugin {
       id = 'elasticsearch.docs-test'
       implementationClass = 'org.elasticsearch.gradle.internal.doc.DocsTestPlugin'
     }
-    esPlugin {
-      id = 'elasticsearch.esplugin'
-      implementationClass = 'org.elasticsearch.gradle.plugin.PluginBuildPlugin'
-    }
     globalBuildInfo {
       id = 'elasticsearch.global-build-info'
       implementationClass = 'org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin'
@@ -89,10 +74,6 @@ gradlePlugin {
       id = 'elasticsearch.internal-distribution-download'
       implementationClass = 'org.elasticsearch.gradle.internal.InternalDistributionDownloadPlugin'
     }
-    internalLicenseheaders {
-      id = 'elasticsearch.internal-licenseheaders'
-      implementationClass = 'org.elasticsearch.gradle.internal.precommit.LicenseHeadersPrecommitPlugin'
-    }
     internalPlugin {
       id = 'elasticsearch.internal-es-plugin'
       implementationClass = 'org.elasticsearch.gradle.internal.InternalPluginBuildPlugin'
@@ -125,14 +106,6 @@ gradlePlugin {
       id = 'elasticsearch.jdk-download'
       implementationClass = 'org.elasticsearch.gradle.internal.JdkDownloadPlugin'
     }
-    publish {
-      id = 'elasticsearch.publish'
-      implementationClass = 'org.elasticsearch.gradle.internal.PublishPlugin'
-    }
-    reaper {
-      id = 'elasticsearch.reaper'
-      implementationClass = 'org.elasticsearch.gradle.ReaperPlugin'
-    }
     repositories {
       id = 'elasticsearch.repositories'
       implementationClass = 'org.elasticsearch.gradle.internal.RepositoriesSetupPlugin'
@@ -169,10 +142,6 @@ gradlePlugin {
       id = 'elasticsearch.test-with-ssl'
       implementationClass = 'org.elasticsearch.gradle.internal.test.TestWithSslPlugin'
     }
-    testclusters {
-      id = 'elasticsearch.testclusters'
-      implementationClass = 'org.elasticsearch.gradle.testclusters.TestClustersPlugin'
-    }
     validateRestSpec {
       id = 'elasticsearch.validate-rest-spec'
       implementationClass = 'org.elasticsearch.gradle.internal.precommit.ValidateRestSpecPlugin'
@@ -187,15 +156,6 @@ gradlePlugin {
     }
   }
 }
-def generateVersionProperties = tasks.register("generateVersionProperties", WriteProperties) {
-  outputFile = "${buildDir}/version.properties"
-  comment = 'Generated version properties'
-  properties(props)
-}
-
-tasks.named("processResources").configure {
-  from(generateVersionProperties)
-}
 
 /*****************************************************************************
  *         Java version                                                      *
@@ -205,11 +165,9 @@ if (JavaVersion.current() < JavaVersion.VERSION_11) {
   throw new GradleException('At least Java 11 is required to build elasticsearch gradle tools')
 }
 
-allprojects {
-  apply plugin: 'java'
-  targetCompatibility = '11'
-  sourceCompatibility = '11'
-}
+def minRuntimeJava = JavaVersion.toVersion(file('src/main/resources/minimumRuntimeVersion').text)
+targetCompatibility = minRuntimeJava
+sourceCompatibility = minRuntimeJava
 
 sourceSets {
   integTest {
@@ -222,6 +180,10 @@ tasks.withType(JavaCompile).configureEach {
   options.encoding = 'UTF-8'
 }
 
+tasks.named('licenseHeaders').configure {
+  // ignore gradle generated binary script plugins
+  excludes << "Elasticsearch*Plugin.java"
+}
 /*****************************************************************************
  *                    Dependencies used by the entire build                  *
  *****************************************************************************/
@@ -238,6 +200,9 @@ dependencies {
   api localGroovy()
   api gradleApi()
 
+  api "org.elasticsearch:build-conventions:$version"
+  api "org.elasticsearch:build-tools:$version"
+
   api 'commons-codec:commons-codec:1.12'
   api 'org.apache.commons:commons-compress:1.19'
   api 'org.apache.ant:ant:1.10.8'
@@ -260,22 +225,26 @@ dependencies {
   api "org.apache.httpcomponents:httpcore:${props.getProperty('httpcore')}"
   compileOnly "com.puppycrawl.tools:checkstyle:${props.getProperty('checkstyle')}"
   testImplementation "com.puppycrawl.tools:checkstyle:${props.getProperty('checkstyle')}"
-  testFixturesApi "junit:junit:${props.getProperty('junit')}"
-  testFixturesApi "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${props.getProperty('randomizedrunner')}"
-  testFixturesApi gradleApi()
-  testFixturesApi gradleTestKit()
+  testImplementation "junit:junit:${props.getProperty('junit')}"
+//  testFixturesApi "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${props.getProperty('randomizedrunner')}"
+//  testFixturesApi gradleApi()
+//  testFixturesApi gradleTestKit()
   testImplementation 'com.github.tomakehurst:wiremock-jre8-standalone:2.23.2'
   testImplementation 'org.mockito:mockito-core:1.9.5'
   testImplementation "org.hamcrest:hamcrest:${props.getProperty('hamcrest')}"
 
+  testImplementation testFixtures("org.elasticsearch:build-tools:$version")
+
   integTestImplementation(platform("org.junit:junit-bom:${props.getProperty('junit5')}"))
   integTestImplementation("org.junit.jupiter:junit-jupiter") {
     because 'allows to write and run Jupiter tests'
   }
+  integTestImplementation("net.bytebuddy:byte-buddy:1.11.0") {
+    because 'Generating dynamic mocks of internal libraries like JdkJarHell'
+  }
   testRuntimeOnly("org.junit.vintage:junit-vintage-engine") {
     because 'allows JUnit 3 and JUnit 4 tests to run'
   }
-
   integTestRuntimeOnly("org.junit.platform:junit-platform-launcher") {
     because 'allows tests to run from IDEs that bundle older version of launcher'
   }
@@ -293,186 +262,12 @@ dependencies {
   integTestImplementation "org.xmlunit:xmlunit-core:2.8.2"
 }
 
-/*****************************************************************************
- *                         Bootstrap repositories                            *
- *****************************************************************************/
-// this will only happen when buildSrc is built on its own during build init
-if (project == rootProject) {
-  apply plugin: 'groovy-gradle-plugin'
-
-  repositories {
-    if (System.getProperty("repos.mavenLocal") != null) {
-      mavenLocal()
-    }
-  }
-  dependencies {
-    // add this so the runtime classpath so Gradle will properly track it as a build runtime classpath input
-    runtimeOnly project('reaper')
-  }
-  // only run tests as build-tools
-  tasks.named("test").configure {
-    enabled = false
-  }
+tasks.register("integTest", Test) {
+  inputs.dir(file("src/testKit")).withPropertyName("testkit dir").withPathSensitivity(PathSensitivity.RELATIVE)
+  systemProperty 'test.version_under_test', version
+  testClassesDirs = sourceSets.integTest.output.classesDirs
+  classpath = sourceSets.integTest.runtimeClasspath
+  useJUnitPlatform()
 }
 
-/*****************************************************************************
- *                           Normal project checks                           *
- *****************************************************************************/
-
-// this happens when included as a normal project in the build, which we do
-// to enforce precommit checks like forbidden apis, as well as setup publishing
-if (project != rootProject) {
-  apply plugin: 'elasticsearch.build'
-  apply plugin: 'elasticsearch.publish'
-
-  // groovydoc succeeds, but has some weird internal exception...
-  tasks.named("groovydoc").configure {
-    enabled = false
-  }
-
-  // build-tools is not ready for primetime with these...
-  tasks.named("dependencyLicenses").configure { enabled = false }
-  tasks.named("dependenciesInfo").configure {enabled = false }
-  tasks.named("dependenciesGraph").configure {enabled = false }
-  disableTasks('forbiddenApisMain', 'forbiddenApisTest', 'forbiddenApisIntegTest', 'forbiddenApisTestFixtures')
-  tasks.named("jarHell").configure {
-    enabled = false
-  }
-  tasks.named("thirdPartyAudit").configure {
-    enabled = false
-  }
-  configurations.register("distribution")
-  configurations.register("reaper")
-
-  dependencies {
-    reaper project('reaper')
-    distribution project(':distribution:archives:windows-zip')
-    distribution project(':distribution:archives:darwin-tar')
-    distribution project(':distribution:archives:darwin-aarch64-tar')
-    distribution project(':distribution:archives:linux-aarch64-tar')
-    distribution project(':distribution:archives:linux-tar')
-    integTestRuntimeOnly(project(":libs:elasticsearch-core"))
-  }
-
-  eclipse {
-    classpath {
-      plusConfigurations += [configurations.integTestRuntimeClasspath]
-    }
-  }
-
-  // for external projects we want to remove the marker file indicating we are running the Elasticsearch project
-  tasks.named("processResources").configure {
-    exclude 'buildSrc.marker'
-    into('META-INF') {
-      from configurations.reaper
-    }
-  }
-
-  tasks.withType(Test).configureEach {
-    // Track reaper jar as a test input using runtime classpath normalization strategy
-    inputs.files(configurations.reaper).withNormalizer(ClasspathNormalizer)
-    useJUnitPlatform()
-  }
-
-  tasks.named("test").configure {
-    include("**/*TestSpec.class")
-  }
-
-  normalization {
-    runtimeClasspath {
-      // We already include the reaper jar as part of our runtime classpath. Ignore the copy in META-INF.
-      ignore('META-INF/reaper.jar')
-    }
-  }
-
-  tasks.named("forbiddenPatterns").configure {
-    exclude '**/*.wav'
-    exclude '**/*.p12'
-    exclude '**/*.jks'
-    exclude '**/*.crt'
-    // the file that actually defines nocommit
-    exclude '**/ForbiddenPatternsTask.java'
-    exclude '**/*.bcfks'
-  }
-
-  tasks.named("testingConventions") {
-    naming.clear()
-    naming {
-      Tests {
-        baseClass 'org.elasticsearch.gradle.internal.test.GradleUnitTestCase'
-      }
-      TestSpec {
-        baseClass 'spock.lang.Specification'
-      }
-    }
-  }
-
-  tasks.register("integTest", Test) {
-    inputs.dir(file("src/testKit")).withPropertyName("testkit dir").withPathSensitivity(PathSensitivity.RELATIVE)
-    systemProperty 'test.version_under_test', version
-    testClassesDirs = sourceSets.integTest.output.classesDirs
-    classpath = sourceSets.integTest.runtimeClasspath
-    useJUnitPlatform()
-  }
-  tasks.named("check").configure { dependsOn("integTest") }
-
-  // for now we hardcode the tests for our build to use the gradle jvm.
-  tasks.withType(Test).configureEach {
-    onlyIf { org.elasticsearch.gradle.internal.info.BuildParams.inFipsJvm == false }
-    it.executable = Jvm.current().getJavaExecutable()
-    maxParallelForks = providers.systemProperty('tests.jvms').forUseAtConfigurationTime().getOrElse(org.elasticsearch.gradle.internal.info.BuildParams.defaultParallel.toString()) as Integer
-  }
-
-  publishing.publications.named("elastic").configure {
-    suppressPomMetadataWarningsFor("testFixturesApiElements")
-    suppressPomMetadataWarningsFor("testFixturesRuntimeElements")
-  }
-}
-
-// Define this here because we need it early.
-class VersionPropertiesLoader {
-  static Properties loadBuildSrcVersion(File input) throws IOException {
-    Properties props = new Properties();
-    InputStream is = new FileInputStream(input)
-    try {
-      props.load(is)
-    } finally {
-      is.close()
-    }
-    loadBuildSrcVersion(props, System.getProperties())
-    return props
-  }
-
-  protected static void loadBuildSrcVersion(Properties loadedProps, Properties systemProperties) {
-    String elasticsearch = loadedProps.getProperty("elasticsearch")
-    if (elasticsearch == null) {
-      throw new IllegalStateException("Elasticsearch version is missing from properties.")
-    }
-    if (elasticsearch.matches("[0-9]+\\.[0-9]+\\.[0-9]+") == false) {
-      throw new IllegalStateException(
-        "Expected elasticsearch version to be numbers only of the form  X.Y.Z but it was: " +
-          elasticsearch
-      )
-    }
-    String qualifier = systemProperties.getProperty("build.version_qualifier", "")
-    if (qualifier.isEmpty() == false) {
-      if (qualifier.matches("(alpha|beta|rc)\\d+") == false) {
-        throw new IllegalStateException("Invalid qualifier: " + qualifier)
-      }
-      elasticsearch += "-" + qualifier
-    }
-    final String buildSnapshotSystemProperty = systemProperties.getProperty("build.snapshot", "true");
-    switch (buildSnapshotSystemProperty) {
-      case "true":
-        elasticsearch += "-SNAPSHOT"
-        break;
-      case "false":
-        // do nothing
-        break;
-      default:
-        throw new IllegalArgumentException(
-          "build.snapshot was set to [" + buildSnapshotSystemProperty + "] but can only be unset or [true|false]");
-    }
-    loadedProps.put("elasticsearch", elasticsearch)
-  }
-}
+tasks.named("check").configure { dependsOn("integTest") }

build-tools-internal/settings.gradle

@@ -0,0 +1,2 @@
+includeBuild '../build-conventions'
+includeBuild '../build-tools'

buildSrc/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGitAwareGradleFuncTest.groovy → build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGitAwareGradleFuncTest.groovy

@@ -9,6 +9,7 @@
 package org.elasticsearch.gradle.fixtures
 
 import org.apache.commons.io.FileUtils
+import org.elasticsearch.gradle.internal.test.InternalAwareGradleRunner
 import org.gradle.testkit.runner.GradleRunner
 import org.junit.Rule
 import org.junit.rules.TemporaryFolder

buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy → build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy

@@ -10,8 +10,10 @@ package org.elasticsearch.gradle.internal
 
 import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
 import org.gradle.testkit.runner.TaskOutcome
+import org.junit.Ignore
 import org.xmlunit.builder.DiffBuilder
 import org.xmlunit.builder.Input
+import spock.lang.IgnoreRest
 
 class PublishPluginFuncTest extends AbstractGradleFuncTest {
 
@@ -38,16 +40,42 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
         file("build/distributions/hello-world-1.0-sources.jar").exists()
         file("build/distributions/hello-world-1.0.pom").exists()
         assertXmlEquals(file("build/distributions/hello-world-1.0.pom").text, """
-            <project xmlns="http://maven.apache.org/POM/4.0.0" 
-                     xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd" 
-                     xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
-              <modelVersion>4.0.0</modelVersion>
-              <groupId>org.acme</groupId>
-              <artifactId>hello-world</artifactId>
-              <version>1.0</version>
-              <name>hello-world</name>
-              <description>custom project description</description>
-            </project>"""
+            <project xmlns="http://maven.apache.org/POM/4.0.0" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <!-- This module was also published with a richer model, Gradle metadata,  -->
+  <!-- which should be used instead. Do not delete the following line which  -->
+  <!-- is to indicate to Gradle or any Gradle module metadata file consumer  -->
+  <!-- that they should prefer consuming it instead. -->
+  <!-- do_not_remove: published-with-gradle-metadata -->
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>org.acme</groupId>
+  <artifactId>hello-world</artifactId>
+  <version>1.0</version>
+  <name>hello-world</name>
+  <description>custom project description</description>
+  <url>unknown</url>
+  <scm>
+    <url>unknown</url>
+  </scm>
+  <inceptionYear>2009</inceptionYear>
+  <licenses>
+    <license>
+      <name>Elastic License 2.0</name>
+      <url>https://raw.githubusercontent.com/elastic/elasticsearch/v1.0/licenses/ELASTIC-LICENSE-2.0.txt</url>
+      <distribution>repo</distribution>
+    </license>
+    <license>
+      <name>Server Side Public License, v 1</name>
+      <url>https://www.mongodb.com/licensing/server-side-public-license</url>
+      <distribution>repo</distribution>
+    </license>
+  </licenses>
+  <developers>
+    <developer>
+      <name>Elastic</name>
+      <url>https://www.elastic.co</url>
+    </developer>
+  </developers>
+</project>"""
         )
     }
 
@@ -99,6 +127,29 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
               <version>1.0</version>
               <name>hello-world</name>
               <description>some description</description>
+              <url>unknown</url>
+              <scm>
+                <url>unknown</url>
+              </scm>
+              <inceptionYear>2009</inceptionYear>
+              <licenses>
+                <license>
+                  <name>Elastic License 2.0</name>
+                  <url>https://raw.githubusercontent.com/elastic/elasticsearch/v1.0/licenses/ELASTIC-LICENSE-2.0.txt</url>
+                  <distribution>repo</distribution>
+                </license>
+                <license>
+                  <name>Server Side Public License, v 1</name>
+                  <url>https://www.mongodb.com/licensing/server-side-public-license</url>
+                  <distribution>repo</distribution>
+                </license>
+              </licenses>
+                <developers>
+                  <developer>
+                    <name>Elastic</name>
+                    <url>https://www.elastic.co</url>
+                  </developer>
+                </developers>
               <dependencies>
                 <dependency>
                   <groupId>org.slf4j</groupId>
@@ -160,6 +211,29 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
               <version>1.0</version>
               <name>hello-world</name>
               <description>some description</description>
+              <url>unknown</url>
+              <scm>
+                <url>unknown</url>
+              </scm>
+              <inceptionYear>2009</inceptionYear>
+              <licenses>
+                <license>
+                  <name>Elastic License 2.0</name>
+                  <url>https://raw.githubusercontent.com/elastic/elasticsearch/v1.0/licenses/ELASTIC-LICENSE-2.0.txt</url>
+                  <distribution>repo</distribution>
+                </license>
+                <license>
+                  <name>Server Side Public License, v 1</name>
+                  <url>https://www.mongodb.com/licensing/server-side-public-license</url>
+                  <distribution>repo</distribution>
+                </license>
+              </licenses>
+              <developers>
+                <developer>
+                  <name>Elastic</name>
+                  <url>https://www.elastic.co</url>
+                </developer>
+              </developers>
               <dependencies>
                 <dependency>
                   <groupId>org.acme</groupId>
@@ -216,15 +290,41 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
         file("build/distributions/hello-world-plugin-1.0-sources.jar").exists()
         file("build/distributions/hello-world-plugin-1.0.pom").exists()
         assertXmlEquals(file("build/distributions/hello-world-plugin-1.0.pom").text, """
-            <project xmlns="http://maven.apache.org/POM/4.0.0" 
-                     xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd" 
-                     xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+            <project xmlns="http://maven.apache.org/POM/4.0.0" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+              <!-- This module was also published with a richer model, Gradle metadata,  -->
+              <!-- which should be used instead. Do not delete the following line which  -->
+              <!-- is to indicate to Gradle or any Gradle module metadata file consumer  -->
+              <!-- that they should prefer consuming it instead. -->
+              <!-- do_not_remove: published-with-gradle-metadata -->
               <modelVersion>4.0.0</modelVersion>
               <groupId>org.acme</groupId>
               <artifactId>hello-world-plugin</artifactId>
               <version>1.0</version>
               <name>hello-world</name>
               <description>custom project description</description>
+              <url>unknown</url>
+              <scm>
+                <url>unknown</url>
+              </scm>
+              <inceptionYear>2009</inceptionYear>
+              <licenses>
+                <license>
+                  <name>Elastic License 2.0</name>
+                  <url>https://raw.githubusercontent.com/elastic/elasticsearch/v1.0/licenses/ELASTIC-LICENSE-2.0.txt</url>
+                  <distribution>repo</distribution>
+                </license>
+                <license>
+                  <name>Server Side Public License, v 1</name>
+                  <url>https://www.mongodb.com/licensing/server-side-public-license</url>
+                  <distribution>repo</distribution>
+                </license>
+              </licenses>
+              <developers>
+                <developer>
+                  <name>Elastic</name>
+                  <url>https://www.elastic.co</url>
+                </developer>
+              </developers>
               <dependencies/>
             </project>"""
         )
@@ -250,7 +350,7 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
             tasks.named('bundlePlugin').configure { enabled = false }
             licenseFile = file('license.txt')
             noticeFile = file('notice.txt')
-            version = "1.0"
+            version = "2.0"
             group = 'org.acme'        
         """
 
@@ -259,22 +359,48 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
 
         then:
         result.task(":generatePom").outcome == TaskOutcome.SUCCESS
-        file("build/distributions/hello-world-plugin-1.0.pom").exists()
-        assertXmlEquals(file("build/distributions/hello-world-plugin-1.0.pom").text, """
-            <project xmlns="http://maven.apache.org/POM/4.0.0" 
-                     xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd" 
-                     xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+        file("build/distributions/hello-world-plugin-2.0.pom").exists()
+        assertXmlEquals(file("build/distributions/hello-world-plugin-2.0.pom").text, """
+            <project xmlns="http://maven.apache.org/POM/4.0.0" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+              <!-- This module was also published with a richer model, Gradle metadata,  -->
+              <!-- which should be used instead. Do not delete the following line which  -->
+              <!-- is to indicate to Gradle or any Gradle module metadata file consumer  -->
+              <!-- that they should prefer consuming it instead. -->
+              <!-- do_not_remove: published-with-gradle-metadata -->
               <modelVersion>4.0.0</modelVersion>
               <groupId>org.acme</groupId>
               <artifactId>hello-world-plugin</artifactId>
-              <version>1.0</version>
+              <version>2.0</version>
               <name>hello-world</name>
               <description>custom project description</description>
+              <url>unknown</url>
+              <scm>
+                <url>unknown</url>
+              </scm>
+              <inceptionYear>2009</inceptionYear>
+              <licenses>
+                <license>
+                  <name>Elastic License 2.0</name>
+                  <url>https://raw.githubusercontent.com/elastic/elasticsearch/v2.0/licenses/ELASTIC-LICENSE-2.0.txt</url>
+                  <distribution>repo</distribution>
+                </license>
+                <license>
+                  <name>Server Side Public License, v 1</name>
+                  <url>https://www.mongodb.com/licensing/server-side-public-license</url>
+                  <distribution>repo</distribution>
+                </license>
+              </licenses>
+              <developers>
+                <developer>
+                  <name>Elastic</name>
+                  <url>https://www.elastic.co</url>
+                </developer>
+              </developers>
             </project>"""
         )
     }
 
-    def "generated pom can be tweaked and validated"() {
+    def "generated pom can be validated"() {
         given:
         // scm info only added for internal builds
         internalBuild()
@@ -288,68 +414,47 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
             group = 'org.acme'        
             description = "just a test project"
             
-            // this is currently required to have validation passed
-            // In our elasticsearch build this is currently setup in the 
-            // root build.gradle file.
-            plugins.withType(MavenPublishPlugin) {
-                publishing {
-                    publications {
-                        // add license information to generated poms
-                        all {
-                            pom.withXml { XmlProvider xml ->
-                                Node node = xml.asNode()
-                                node.appendNode('inceptionYear', '2009')
-                
-                                Node license = node.appendNode('licenses').appendNode('license')
-                                license.appendNode('name', "The Apache Software License, Version 2.0")
-                                license.appendNode('url', "http://www.apache.org/licenses/LICENSE-2.0.txt")
-                                license.appendNode('distribution', 'repo')
-              
-                                Node developer = node.appendNode('developers').appendNode('developer')
-                                developer.appendNode('name', 'Elastic')
-                                developer.appendNode('url', 'https://www.elastic.co')
-                            }
-                        }
-                    }
-                }
-            }
+            ext.projectLicenses.set(['The Apache Software License, Version 2.0': 'http://www.apache.org/licenses/LICENSE-2.0'])
         """
 
         when:
-        def result = gradleRunner('generatePom', 'validatElasticPom').build()
+        def result = gradleRunner('generatePom', 'validateElasticPom').build()
 
         then:
         result.task(":generatePom").outcome == TaskOutcome.SUCCESS
         file("build/distributions/hello-world-1.0.pom").exists()
         assertXmlEquals(file("build/distributions/hello-world-1.0.pom").text, """
-            <project xmlns="http://maven.apache.org/POM/4.0.0" 
-                     xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd" 
-                     xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
-              <modelVersion>4.0.0</modelVersion>
-              <groupId>org.acme</groupId>
-              <artifactId>hello-world</artifactId>
-              <version>1.0</version>
-              <name>hello-world</name>
-              <description>just a test project</description>
-              <url>https://some-repo.com/repo.git</url>
-              <scm>
-                <url>https://some-repo.com/repo.git</url>
-              </scm>
-              <inceptionYear>2009</inceptionYear>
-              <licenses>
-                <license>
-                  <name>The Apache Software License, Version 2.0</name>
-                  <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
-                  <distribution>repo</distribution>
-                </license>
-              </licenses>
-              <developers>
-                <developer>
-                  <name>Elastic</name>
-                  <url>https://www.elastic.co</url>
-                </developer>
-              </developers>
-            </project>"""
+            <project xmlns="http://maven.apache.org/POM/4.0.0" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+          <!-- This module was also published with a richer model, Gradle metadata,  -->
+          <!-- which should be used instead. Do not delete the following line which  -->
+          <!-- is to indicate to Gradle or any Gradle module metadata file consumer  -->
+          <!-- that they should prefer consuming it instead. -->
+          <!-- do_not_remove: published-with-gradle-metadata -->
+          <modelVersion>4.0.0</modelVersion>
+          <groupId>org.acme</groupId>
+          <artifactId>hello-world</artifactId>
+          <version>1.0</version>
+          <name>hello-world</name>
+          <description>just a test project</description>
+          <url>unknown</url>
+          <scm>
+            <url>unknown</url>
+          </scm>
+          <inceptionYear>2009</inceptionYear>
+          <licenses>
+            <license>
+              <name>The Apache Software License, Version 2.0</name>
+              <url>http://www.apache.org/licenses/LICENSE-2.0</url>
+              <distribution>repo</distribution>
+            </license>
+          </licenses>
+          <developers>
+            <developer>
+              <name>Elastic</name>
+              <url>https://www.elastic.co</url>
+            </developer>
+          </developers>
+        </project>"""
         )
     }
 

buildSrc/src/integTest/java/org/elasticsearch/gradle/internal/BuildPluginIT.java → build-tools-internal/src/integTest/java/org/elasticsearch/gradle/internal/BuildPluginIT.java

@@ -46,11 +46,8 @@ public class BuildPluginIT extends GradleIntegrationTestCase {
 
     public void testLicenseAndNotice() throws IOException {
         BuildResult result = getGradleRunner().withArguments("clean", "assemble").build();
-
         assertTaskSuccessful(result, ":assemble");
-
         assertBuildFileExists(result, projectName(), "distributions/elasticsearch.build.jar");
-
         try (ZipFile zipFile = new ZipFile(new File(getBuildDir(projectName()), "distributions/elasticsearch.build.jar"))) {
             ZipEntry licenseEntry = zipFile.getEntry("META-INF/LICENSE.txt");
             ZipEntry noticeEntry = zipFile.getEntry("META-INF/NOTICE.txt");

build-tools-internal/src/integTest/java/org/elasticsearch/gradle/internal/test/TestClasspathUtils.java

@@ -0,0 +1,82 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.test;
+
+import java.io.File;
+import java.io.IOException;
+
+import net.bytebuddy.ByteBuddy;
+import net.bytebuddy.description.modifier.Ownership;
+import net.bytebuddy.description.modifier.Visibility;
+import net.bytebuddy.description.type.TypeDescription;
+import net.bytebuddy.dynamic.DynamicType;
+import net.bytebuddy.dynamic.scaffold.InstrumentedType;
+import net.bytebuddy.implementation.ExceptionMethod;
+import net.bytebuddy.implementation.FixedValue;
+import net.bytebuddy.implementation.Implementation;
+import net.bytebuddy.implementation.bytecode.ByteCodeAppender;
+
+import static org.junit.Assert.fail;
+
+public class TestClasspathUtils {
+
+    public static void setupJarJdkClasspath(File projectRoot) {
+        generateJdkJarHellCheck(projectRoot, FixedValue.value(TypeDescription.VOID));
+    }
+
+    public static void setupJarJdkClasspath(File projectRoot, String errorMessage) {
+        generateJdkJarHellCheck(projectRoot,
+                ExceptionMethod.throwing(IllegalStateException.class, errorMessage));
+    }
+
+    private static void generateJdkJarHellCheck(File projectRoot, Implementation mainImplementation) {
+        DynamicType.Unloaded<?> dynamicType = new ByteBuddy()
+                    .subclass(Object.class)
+                    .name("org.elasticsearch.bootstrap.JdkJarHellCheck")
+                .defineMethod("main",   void.class, Visibility.PUBLIC, Ownership.STATIC)
+                .withParameters(String[].class)
+                .intercept(mainImplementation)
+                .make();
+        try {
+            dynamicType.toJar(targetFile(projectRoot));
+        } catch (IOException e) {
+            e.printStackTrace();
+            fail("Cannot setup jdk jar hell classpath");
+        }
+    }
+
+    private static File targetFile(File projectRoot) {
+        File targetFile = new File(
+                projectRoot,
+                "sample_jars/build/testrepo/org/elasticsearch/elasticsearch-core/current/elasticsearch-core-current.jar"
+        );
+
+        targetFile.getParentFile().mkdirs();
+        return targetFile;
+    }
+
+
+    private static class InconsistentParameterReferenceMethod implements net.bytebuddy.implementation.Implementation {
+        @Override
+        public ByteCodeAppender appender(Target implementationTarget) {
+            return null;
+        }
+
+        @Override
+        public InstrumentedType prepare(InstrumentedType instrumentedType) {
+            return null;
+        }
+    }
+
+    public static class JdkJarHellBase {
+        public static void main(String[] args) {
+            System.out.println("args = " + args);
+        }
+    }
+}

buildSrc/src/integTest/java/org/elasticsearch/gradle/precommit/TestingConventionsTasksIT.java → build-tools-internal/src/integTest/java/org/elasticsearch/gradle/jarhell/TestingConventionsTasksIT.java

@@ -5,7 +5,7 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-package org.elasticsearch.gradle.precommit;
+package org.elasticsearch.gradle.jarhell;
 
 import org.elasticsearch.gradle.internal.test.GradleIntegrationTestCase;
 import org.gradle.testkit.runner.BuildResult;

buildSrc/src/integTest/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTaskIT.java → build-tools-internal/src/integTest/java/org/elasticsearch/gradle/jarhell/ThirdPartyAuditTaskIT.java

@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-package org.elasticsearch.gradle.precommit;
+package org.elasticsearch.gradle.jarhell;
 
 import org.elasticsearch.gradle.internal.test.GradleIntegrationTestCase;
 import org.gradle.testkit.runner.BuildResult;
@@ -26,32 +26,33 @@ public class ThirdPartyAuditTaskIT extends GradleIntegrationTestCase {
         // Build the sample jars
         getGradleRunner().withArguments(":sample_jars:build", "-s").build();
         // propagate jdkjarhell jar
-        setupJarJdkClasspath(getProjectDir());
     }
 
     public void testElasticsearchIgnored() {
         BuildResult result = getGradleRunner().withArguments(
-            ":clean",
-            ":empty",
-            "-s",
-            "-PcompileOnlyGroup=elasticsearch.gradle:broken-log4j",
-            "-PcompileOnlyVersion=0.0.1",
-            "-PcompileGroup=elasticsearch.gradle:dummy-io",
-            "-PcompileVersion=0.0.1"
+                ":clean",
+                ":empty",
+                "-s",
+                "-PcompileOnlyGroup=elasticsearch.gradle:broken-log4j",
+                "-PcompileOnlyVersion=0.0.1",
+                "-PcompileGroup=elasticsearch.gradle:dummy-io",
+                "-PcompileVersion=0.0.1"
         ).build();
         assertTaskNoSource(result, ":empty");
         assertNoDeprecationWarning(result);
     }
 
     public void testViolationFoundAndCompileOnlyIgnored() {
+        setupJarJdkClasspath(getProjectDir());
+
         BuildResult result = getGradleRunner().withArguments(
-            ":clean",
-            ":absurd",
-            "-s",
-            "-PcompileOnlyGroup=other.gradle:broken-log4j",
-            "-PcompileOnlyVersion=0.0.1",
-            "-PcompileGroup=other.gradle:dummy-io",
-            "-PcompileVersion=0.0.1"
+                ":clean",
+                ":absurd",
+                "-s",
+                "-PcompileOnlyGroup=other.gradle:broken-log4j",
+                "-PcompileOnlyVersion=0.0.1",
+                "-PcompileGroup=other.gradle:dummy-io",
+                "-PcompileVersion=0.0.1"
         ).buildAndFail();
 
         assertTaskFailed(result, ":absurd");
@@ -61,44 +62,49 @@ public class ThirdPartyAuditTaskIT extends GradleIntegrationTestCase {
     }
 
     public void testClassNotFoundAndCompileOnlyIgnored() {
+        setupJarJdkClasspath(getProjectDir());
         BuildResult result = getGradleRunner().withArguments(
-            ":clean",
-            ":absurd",
-            "-s",
-            "-PcompileGroup=other.gradle:broken-log4j",
-            "-PcompileVersion=0.0.1",
-            "-PcompileOnlyGroup=other.gradle:dummy-io",
-            "-PcompileOnlyVersion=0.0.1"
+                ":clean",
+                ":absurd",
+                "-s",
+                "-PcompileGroup=other.gradle:broken-log4j",
+                "-PcompileVersion=0.0.1",
+                "-PcompileOnlyGroup=other.gradle:dummy-io",
+                "-PcompileOnlyVersion=0.0.1"
         ).buildAndFail();
         assertTaskFailed(result, ":absurd");
 
         assertOutputContains(
-            result.getOutput(),
-            "Missing classes:",
-            "  * org.apache.logging.log4j.LogManager",
-            "> Audit of third party dependencies failed"
+                result.getOutput(),
+                "Missing classes:",
+                "  * org.apache.logging.log4j.LogManager",
+                "> Audit of third party dependencies failed"
         );
         assertOutputMissing(result.getOutput(), "Classes with violations:");
         assertNoDeprecationWarning(result);
     }
 
     public void testJarHellWithJDK() {
+        setupJarJdkClasspath(getProjectDir(), "> Audit of third party dependencies failed:" +
+                "   Jar Hell with the JDK:" +
+                "    * java.lang.String"
+        );
         BuildResult result = getGradleRunner().withArguments(
-            ":clean",
-            ":absurd",
-            "-s",
-            "-PcompileGroup=other.gradle:jarhellJdk",
-            "-PcompileVersion=0.0.1",
-            "-PcompileOnlyGroup=other.gradle:dummy-io",
-            "-PcompileOnlyVersion=0.0.1"
+                ":clean",
+                ":absurd",
+                "-s",
+                "-PcompileGroup=other.gradle:jarhellJdk",
+                "-PcompileVersion=0.0.1",
+                "-PcompileOnlyGroup=other.gradle:dummy-io",
+                "-PcompileOnlyVersion=0.0.1"
         ).buildAndFail();
         assertTaskFailed(result, ":absurd");
 
         assertOutputContains(
-            result.getOutput(),
-            "> Audit of third party dependencies failed:",
-            "   Jar Hell with the JDK:",
-            "    * java.lang.String"
+                result.getOutput(),
+                "> Audit of third party dependencies failed:",
+                "   Jar Hell with the JDK:",
+                "    * java.lang.String"
         );
         assertOutputMissing(result.getOutput(), "Classes with violations:");
         assertNoDeprecationWarning(result);
@@ -106,13 +112,13 @@ public class ThirdPartyAuditTaskIT extends GradleIntegrationTestCase {
 
     public void testElasticsearchIgnoredWithViolations() {
         BuildResult result = getGradleRunner().withArguments(
-            ":clean",
-            ":absurd",
-            "-s",
-            "-PcompileOnlyGroup=elasticsearch.gradle:broken-log4j",
-            "-PcompileOnlyVersion=0.0.1",
-            "-PcompileGroup=elasticsearch.gradle:dummy-io",
-            "-PcompileVersion=0.0.1"
+                ":clean",
+                ":absurd",
+                "-s",
+                "-PcompileOnlyGroup=elasticsearch.gradle:broken-log4j",
+                "-PcompileOnlyVersion=0.0.1",
+                "-PcompileGroup=elasticsearch.gradle:dummy-io",
+                "-PcompileVersion=0.0.1"
         ).build();
         assertTaskNoSource(result, ":absurd");
         assertNoDeprecationWarning(result);

buildSrc/src/main/groovy/elasticsearch.formatting.gradle → build-tools-internal/src/main/groovy/elasticsearch.formatting.gradle

@@ -165,7 +165,7 @@ subprojects {
           target 'src/**/*.java'
 
           removeUnusedImports()
-          eclipse().configFile rootProject.file('buildSrc/formatterConfig.xml')
+          eclipse().configFile rootProject.file('build-tools-internal/formatterConfig.xml')
           trimTrailingWhitespace()
 
           // See CONTRIBUTING.md for details of when to enabled this.

buildSrc/src/main/groovy/elasticsearch.ide.gradle → build-tools-internal/src/main/groovy/elasticsearch.ide.gradle

@@ -13,17 +13,6 @@ import java.nio.file.Files
 import java.nio.file.Paths
 import java.nio.file.StandardCopyOption
 
-buildscript {
-  repositories {
-    maven {
-      url "https://plugins.gradle.org/m2/"
-    }
-  }
-  dependencies {
-    classpath "gradle.plugin.org.jetbrains.gradle.plugin.idea-ext:gradle-idea-ext:0.7"
-  }
-}
-
 allprojects {
   apply plugin: 'idea'
 
@@ -36,9 +25,9 @@ tasks.register('configureIdeCheckstyle') {
   group = 'ide'
   description = 'Generated a suitable checkstyle config for IDEs'
 
-  String checkstyleConfig = 'buildSrc/src/main/resources/checkstyle.xml'
-  String checkstyleSuppressions = 'buildSrc/src/main/resources/checkstyle_suppressions.xml'
-  String checkstyleIdeFragment = 'buildSrc/src/main/resources/checkstyle_ide_fragment.xml'
+  String checkstyleConfig = 'build-tools-internal/src/main/resources/checkstyle.xml'
+  String checkstyleSuppressions = 'build-tools-internal/src/main/resources/checkstyle_suppressions.xml'
+  String checkstyleIdeFragment = 'build-tools-internal/src/main/resources/checkstyle_ide_fragment.xml'
   String checkstyleIdeConfig = "$rootDir/checkstyle_ide.xml"
 
   inputs.files(file(checkstyleConfig), file(checkstyleIdeFragment))

buildSrc/src/main/groovy/elasticsearch.runtime-jdk-provision.gradle → build-tools-internal/src/main/groovy/elasticsearch.runtime-jdk-provision.gradle

@@ -24,7 +24,7 @@ jdks {
     }
 }
 
-configure(allprojects - project(':build-tools')) {
+configure(allprojects) {
     project.tasks.withType(Test).configureEach { Test test ->
         if (BuildParams.getIsRuntimeJavaHomeSet()) {
             test.executable = "${BuildParams.runtimeJavaHome}/bin/java"

buildSrc/src/main/groovy/org/elasticsearch/gradle/internal/AntFixtureStop.groovy → build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/AntFixtureStop.groovy

@@ -9,7 +9,6 @@
 package org.elasticsearch.gradle.internal
 
 import org.apache.tools.ant.taskdefs.condition.Os
-import org.elasticsearch.gradle.FixtureStop
 import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.internal.test.AntFixture
 import org.gradle.api.file.FileSystemOperations