Port LicenseHeadersTask to java and remove ant layer (#65310)

This ports Licenseheaders task from using groovies antbuilder to rely on the rat ant task 
to use rat directly which allows us to port this completely to java. We also refactored rat to 
generate xml instead of plain report file. 

- More efficient as plain file generation is based on running an xslt transformation on top 
of the xml output (via ant rat in a different thread which results in faster task execution as seen 
in the attached profiler report)
- Easier machine readable
- By using rat directly we can avoid parsing the log file for reporting.
- Ported more buildSrc sources from groovy to java which has been blocked from porting by 
LicenseHeadersTask

buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/LicenseHeadersPrecommitPluginFuncTest.groovy

@@ -0,0 +1,143 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle.internal.precommit
+
+import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
+import org.gradle.testkit.runner.TaskOutcome
+
+class LicenseHeadersPrecommitPluginFuncTest extends AbstractGradleFuncTest {
+
+    def "detects invalid files with invalid license header"() {
+        given:
+        buildFile << """
+        plugins {
+            id 'java'
+            id 'elasticsearch.internal-licenseheaders'
+        }
+        """
+        apacheSourceFile()
+        unknownSourceFile()
+        unapprovedSourceFile()
+
+        when:
+        def result = gradleRunner("licenseHeaders").buildAndFail()
+
+        then:
+        result.task(":licenseHeaders").outcome == TaskOutcome.FAILED
+        assertOutputContains(result.output, "> License header problems were found! Full details: ./build/reports/licenseHeaders/rat.xml")
+        assertOutputContains(result.output, "./src/main/java/org/acme/UnknownLicensed.java")
+        assertOutputContains(result.output, "./src/main/java/org/acme/UnapprovedLicensed.java")
+        normalizedOutput(result.output).contains("./src/main/java/org/acme/ApacheLicensed.java") == false
+    }
+
+    def "can filter source files"() {
+        given:
+        buildFile << """
+        plugins {
+            id 'java'
+            id 'elasticsearch.internal-licenseheaders'
+        }
+        
+        tasks.named("licenseHeaders").configure {
+            excludes << 'org/acme/filtered/**/*'
+        }
+        """
+        apacheSourceFile()
+        unknownSourceFile("src/main/java/org/acme/filtered/FilteredUnknownLicensed.java")
+        unapprovedSourceFile("src/main/java/org/acme/filtered/FilteredUnapprovedLicensed.java")
+
+        when:
+        def result = gradleRunner("licenseHeaders").build()
+
+        then:
+        result.task(":licenseHeaders").outcome == TaskOutcome.SUCCESS
+    }
+
+    private File unapprovedSourceFile(String filePath = "src/main/java/org/acme/UnapprovedLicensed.java") {
+        File sourceFile = file(filePath);
+        sourceFile << """
+/*
+ * Copyright (C) 2007 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package ${packageString(sourceFile)};
+ 
+ public class ${sourceFile.getName() - ".java"} {
+ }
+ """
+    }
+
+    private File unknownSourceFile(String filePath = "src/main/java/org/acme/UnknownLicensed.java") {
+        File sourceFile = file(filePath);
+        sourceFile << """
+/*
+ * Blubb my custom license shrug!
+ */
+ 
+ package ${packageString(sourceFile)};
+ 
+ public class ${sourceFile.getName() - ".java"} {
+ }
+ """
+    }
+
+    private File apacheSourceFile() {
+        file("src/main/java/org/acme/ApacheLicensed.java") << """
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ 
+ package org.acme; 
+ public class ApacheLicensed {
+ }
+ """
+    }
+
+    private static String packageString(File sourceFile) {
+        (sourceFile.getPath().substring(sourceFile.getPath().indexOf("src/main/java")) - "src/main/java/" - ("/" + sourceFile.getName())).replaceAll("/", ".")
+    }
+}

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -1,100 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.gradle
-
-import groovy.transform.CompileStatic
-import org.elasticsearch.gradle.info.BuildParams
-import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin
-import org.elasticsearch.gradle.internal.InternalPlugin
-import org.elasticsearch.gradle.internal.precommit.InternalPrecommitTasks
-import org.elasticsearch.gradle.precommit.PrecommitTasks
-import org.gradle.api.GradleException
-import org.gradle.api.InvalidUserDataException
-import org.gradle.api.Plugin
-import org.gradle.api.Project
-import org.gradle.api.file.CopySpec
-import org.gradle.api.plugins.ExtraPropertiesExtension
-import org.gradle.api.tasks.bundling.Jar
-
-/**
- * Encapsulates build configuration for elasticsearch projects.
- */
-@CompileStatic
-class BuildPlugin implements Plugin<Project> {
-
-    @Override
-    void apply(Project project) {
-        // make sure the global build info plugin is applied to the root project
-        project.rootProject.pluginManager.apply(GlobalBuildInfoPlugin)
-        checkExternalInternalPluginUsages(project);
-
-        if (project.pluginManager.hasPlugin('elasticsearch.standalone-rest-test')) {
-            throw new InvalidUserDataException('elasticsearch.standalone-test, '
-                    + 'elasticsearch.standalone-rest-test, and elasticsearch.build '
-                    + 'are mutually exclusive')
-        }
-        project.pluginManager.apply('elasticsearch.java')
-        configureLicenseAndNotice(project)
-        project.pluginManager.apply('elasticsearch.publish')
-        project.pluginManager.apply(DependenciesInfoPlugin)
-        project.pluginManager.apply(DependenciesGraphPlugin)
-
-        BuildParams.withInternalBuild {
-            InternalPrecommitTasks.create(project, true)
-        }.orElse {
-            PrecommitTasks.create(project)
-        }
-    }
-
-    private static void checkExternalInternalPluginUsages(Project project) {
-        if (BuildParams.isInternal() == false) {
-            project.getPlugins().withType(InternalPlugin.class) { InternalPlugin internalPlugin ->
-                throw new GradleException(internalPlugin.getExternalUseErrorMessage())
-            }
-        }
-    }
-
-    static void configureLicenseAndNotice(Project project) {
-        ExtraPropertiesExtension ext = project.extensions.getByType(ExtraPropertiesExtension)
-        ext.set('licenseFile',  null)
-        ext.set('noticeFile', null)
-        // add license/notice files
-        project.afterEvaluate {
-            project.tasks.withType(Jar).configureEach { Jar jarTask ->
-                if (ext.has('licenseFile') == false || ext.get('licenseFile') == null || ext.has('noticeFile') == false || ext.get('noticeFile') == null) {
-                    throw new GradleException("Must specify license and notice file for project ${project.path}")
-                }
-
-                File licenseFile = ext.get('licenseFile') as File
-                File noticeFile = ext.get('noticeFile') as File
-
-                jarTask.metaInf { CopySpec spec ->
-                    spec.from(licenseFile.parent) { CopySpec from ->
-                        from.include licenseFile.name
-                        from.rename { 'LICENSE.txt' }
-                    }
-                    spec.from(noticeFile.parent) { CopySpec from ->
-                        from.include noticeFile.name
-                        from.rename { 'NOTICE.txt' }
-                    }
-                }
-            }
-        }
-    }
-}

buildSrc/src/main/groovy/org/elasticsearch/gradle/DependenciesInfoTask.groovy

@@ -1,180 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.gradle
-
-import org.elasticsearch.gradle.internal.precommit.DependencyLicensesTask
-import org.elasticsearch.gradle.precommit.LicenseAnalyzer
-import org.gradle.api.artifacts.Configuration
-import org.gradle.api.artifacts.Dependency
-import org.gradle.api.artifacts.DependencySet
-import org.gradle.api.artifacts.ProjectDependency
-import org.gradle.api.internal.ConventionTask
-import org.gradle.api.tasks.Input
-import org.gradle.api.tasks.InputDirectory
-import org.gradle.api.tasks.InputFiles
-import org.gradle.api.tasks.Optional
-import org.gradle.api.tasks.OutputFile
-import org.gradle.api.tasks.TaskAction
-
-/**
- * A task to gather information about the dependencies and export them into a csv file.
- *
- * The following information is gathered:
- * <ul>
- *     <li>name: name that identifies the library (groupId:artifactId)</li>
- *     <li>version</li>
- *     <li>URL: link to have more information about the dependency.</li>
- *     <li>license: <a href="https://spdx.org/licenses/">SPDX license</a> identifier, custom license or UNKNOWN.</li>
- * </ul>
- *
- */
-class DependenciesInfoTask extends ConventionTask {
-
-    /** Dependencies to gather information from. */
-    @InputFiles
-    Configuration runtimeConfiguration
-
-    /** We subtract compile-only dependencies. */
-    @InputFiles
-    Configuration compileOnlyConfiguration
-
-    /** Directory to read license files */
-    @Optional
-    @InputDirectory
-    File licensesDir = new File(project.projectDir, 'licenses').exists() ? new File(project.projectDir, 'licenses') : null
-
-    @OutputFile
-    File outputFile = new File(project.buildDir, "reports/dependencies/dependencies.csv")
-
-    private LinkedHashMap<String, String> mappings
-
-    DependenciesInfoTask() {
-        description = 'Create a CSV file with dependencies information.'
-    }
-
-    @TaskAction
-    void generateDependenciesInfo() {
-
-        final DependencySet runtimeDependencies = runtimeConfiguration.getAllDependencies()
-        // we have to resolve the transitive dependencies and create a group:artifactId:version map
-        final Set<String> compileOnlyArtifacts =
-                compileOnlyConfiguration
-                        .getResolvedConfiguration()
-                        .resolvedArtifacts
-                        .collect { it -> "${it.moduleVersion.id.group}:${it.moduleVersion.id.name}:${it.moduleVersion.id.version}" }
-
-        final StringBuilder output = new StringBuilder()
-
-        for (final Dependency dependency : runtimeDependencies) {
-            // we do not need compile-only dependencies here
-            if (compileOnlyArtifacts.contains("${dependency.group}:${dependency.name}:${dependency.version}")) {
-                continue
-            }
-            // only external dependencies are checked
-            if (dependency instanceof ProjectDependency) {
-                continue
-            }
-
-            final String url = createURL(dependency.group, dependency.name, dependency.version)
-            final String dependencyName = DependencyLicensesTask.getDependencyName(getMappings(), dependency.name)
-            logger.info("mapped dependency ${dependency.group}:${dependency.name} to ${dependencyName} for license info")
-
-            final String licenseType = getLicenseType(dependency.group, dependencyName)
-            output.append("${dependency.group}:${dependency.name},${dependency.version},${url},${licenseType}\n")
-
-        }
-        outputFile.setText(output.toString(), 'UTF-8')
-    }
-
-    @Input
-    LinkedHashMap<String, String> getMappings() {
-        return mappings
-    }
-
-    void setMappings(LinkedHashMap<String, String> mappings) {
-        this.mappings = mappings
-    }
-
-    /**
-     * Create an URL on <a href="https://repo1.maven.org/maven2/">Maven Central</a>
-     * based on dependency coordinates.
-     */
-    protected String createURL(final String group, final String name, final String version){
-        final String baseURL = 'https://repo1.maven.org/maven2'
-        return "${baseURL}/${group.replaceAll('\\.' , '/')}/${name}/${version}"
-    }
-
-    /**
-     * Read the LICENSE file associated with the dependency and determine a license type.
-     *
-     * The license type is one of the following values:
-     * <u>
-     *     <li><em>UNKNOWN</em> if LICENSE file is not present for this dependency.</li>
-     *     <li><em>one SPDX identifier</em> if the LICENSE content matches with an SPDX license.</li>
-     *     <li><em>Custom;URL</em> if it's not an SPDX license,
-     *          URL is the Github URL to the LICENSE file in elasticsearch repository.</li>
-     * </ul>
-     *
-     * @param group dependency group
-     * @param name dependency name
-     * @return SPDX identifier, UNKNOWN or a Custom license
-     */
-    protected String getLicenseType(final String group, final String name) {
-        File license = getDependencyInfoFile(group, name, 'LICENSE')
-        String licenseType
-
-        final LicenseAnalyzer.LicenseInfo licenseInfo = LicenseAnalyzer.licenseType(license)
-        if (licenseInfo.spdxLicense == false) {
-            // License has not be identified as SPDX.
-            // As we have the license file, we create a Custom entry with the URL to this license file.
-            final gitBranch = System.getProperty('build.branch', 'master')
-            final String githubBaseURL = "https://raw.githubusercontent.com/elastic/elasticsearch/${gitBranch}/"
-            licenseType = "${licenseInfo.identifier};${license.getCanonicalPath().replaceFirst('.*/elasticsearch/', githubBaseURL)},"
-        } else {
-            licenseType = "${licenseInfo.identifier},"
-        }
-
-        if (licenseInfo.sourceRedistributionRequired) {
-            File sources = getDependencyInfoFile(group, name, 'SOURCES')
-            licenseType += "${sources.text.trim()}"
-        }
-
-        return licenseType
-    }
-
-    protected File getDependencyInfoFile(final String group, final String name, final String infoFileSuffix) {
-        File license = null
-
-        if (licensesDir != null) {
-            licensesDir.eachFileMatch({ it ==~ /.*-${infoFileSuffix}.*/ }) { File file ->
-                String prefix = file.name.split("-${infoFileSuffix}.*")[0]
-                if (group.contains(prefix) || name.contains(prefix)) {
-                    license = file.getAbsoluteFile()
-                }
-            }
-        }
-
-        if (license == null) {
-            throw new IllegalStateException("Unable to find ${infoFileSuffix} file for dependency ${group}:${name} in ${licensesDir}")
-        }
-
-        return license
-    }
-}

buildSrc/src/main/groovy/org/elasticsearch/gradle/internal/precommit/LicenseHeadersPrecommitPlugin.groovy

@@ -1,53 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.gradle.internal.precommit
-
-import org.elasticsearch.gradle.internal.InternalPlugin
-import org.elasticsearch.gradle.precommit.PrecommitPlugin
-import org.elasticsearch.gradle.util.GradleUtils
-import org.gradle.api.Project
-import org.gradle.api.Task
-import org.gradle.api.provider.ProviderFactory
-import org.gradle.api.tasks.SourceSetContainer
-import org.gradle.api.tasks.TaskProvider
-
-import javax.inject.Inject
-
-class LicenseHeadersPrecommitPlugin extends PrecommitPlugin implements InternalPlugin {
-
-    private ProviderFactory providerFactory
-
-    @Inject
-    LicenseHeadersPrecommitPlugin(ProviderFactory providerFactory) {
-        this.providerFactory = providerFactory
-    }
-
-    @Override
-    TaskProvider<? extends Task> createTask(Project project) {
-        return project.getTasks().register("licenseHeaders", LicenseHeadersTask.class) {
-            SourceSetContainer sourceSets = GradleUtils.getJavaSourceSets(getProject());
-            it.getSourceFolders().addAll(
-                    providerFactory.provider() {
-                        return sourceSets.collect { it.allJava }.flatten()
-                    }
-            )
-        }
-    }
-}

buildSrc/src/main/groovy/org/elasticsearch/gradle/internal/precommit/LicenseHeadersTask.groovy

@@ -1,186 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.gradle.internal.precommit
-
-import org.apache.rat.anttasks.Report
-import org.apache.rat.anttasks.SubstringLicenseMatcher
-import org.apache.rat.license.SimpleLicenseFamily
-import org.elasticsearch.gradle.AntTask
-import org.gradle.api.file.FileCollection
-import org.gradle.api.provider.ListProperty
-import org.gradle.api.tasks.Input
-import org.gradle.api.tasks.InputFiles
-import org.gradle.api.tasks.Internal
-import org.gradle.api.tasks.OutputFile
-import org.gradle.api.tasks.SkipWhenEmpty
-
-import java.nio.file.Files
-
-/**
- * Checks files for license headers.
- * <p>
- * This is a port of the apache lucene check
- */
-abstract class LicenseHeadersTask extends AntTask {
-
-    @OutputFile
-    File reportFile = new File(project.buildDir, 'reports/licenseHeaders/rat.log')
-
-    /** Allowed license families for this project. */
-    @Input
-    List<String> approvedLicenses = ['Apache', 'Generated', 'Vendored']
-
-    /**
-     * Files that should be excluded from the license header check. Use with extreme care, only in situations where the license on the
-     * source file is compatible with the codebase but we do not want to add the license to the list of approved headers (to avoid the
-     * possibility of inadvertently using the license on our own source files).
-     */
-    @Input
-    List<String> excludes = []
-
-    /**
-     * Additional license families that may be found. The key is the license category name (5 characters),
-     * followed by the family name and the value list of patterns to search for.
-     */
-    protected Map<String, String> additionalLicenses = new HashMap<>()
-
-    LicenseHeadersTask() {
-        description = "Checks sources for missing, incorrect, or unacceptable license headers"
-    }
-
-    /**
-     * The list of java files to check. protected so the afterEvaluate closure in the
-     * constructor can write to it.
-     */
-    @InputFiles
-    @SkipWhenEmpty
-    List<FileCollection> getJavaFiles() {
-        return sourceFolders.get()
-    }
-
-    /**
-     * Add a new license type.
-     *
-     * The license may be added to the {@link #approvedLicenses} using the {@code familyName}.
-     *
-     * @param categoryName A 5-character string identifier for the license
-     * @param familyName An expanded string name for the license
-     * @param pattern A pattern to search for, which if found, indicates a file contains the license
-     */
-    void additionalLicense(String categoryName, String familyName, String pattern) {
-        if (categoryName.length() != 5) {
-            throw new IllegalArgumentException("License category name must be exactly 5 characters, got ${categoryName}");
-        }
-        additionalLicenses.put(categoryName + familyName, pattern);
-    }
-
-    @Override
-    protected void runAnt(AntBuilder ant) {
-        ant.project.addTaskDefinition('ratReport', Report)
-        ant.project.addDataTypeDefinition('substringMatcher', SubstringLicenseMatcher)
-        ant.project.addDataTypeDefinition('approvedLicense', SimpleLicenseFamily)
-
-        Files.deleteIfExists(reportFile.toPath())
-
-        // run rat, going to the file
-        ant.ratReport(reportFile: reportFile.absolutePath, addDefaultLicenseMatchers: true) {
-            for (FileCollection dirSet : javaFiles) {
-                for (File dir : dirSet.srcDirs) {
-                    // sometimes these dirs don't exist, e.g. site-plugin has no actual java src/main...
-                    if (dir.exists()) {
-                        ant.fileset(dir: dir, excludes: excludes.join(' '))
-                    }
-                }
-            }
-
-            // BSD 4-clause stuff (is disallowed below)
-            // we keep this here, in case someone adds BSD code for some reason, it should never be allowed.
-            substringMatcher(licenseFamilyCategory: "BSD4 ",
-                    licenseFamilyName: "Original BSD License (with advertising clause)") {
-                pattern(substring: "All advertising materials")
-            }
-
-            // Apache
-            substringMatcher(licenseFamilyCategory: "AL   ",
-                    licenseFamilyName: "Apache") {
-                // Apache license (ES)
-                pattern(substring: "Licensed to Elasticsearch under one or more contributor")
-            }
-
-            // Generated resources
-            substringMatcher(licenseFamilyCategory: "GEN  ",
-                    licenseFamilyName: "Generated") {
-                // parsers generated by antlr
-                pattern(substring: "ANTLR GENERATED CODE")
-            }
-
-            // Vendored Code
-            substringMatcher(licenseFamilyCategory: "VEN  ",
-                    licenseFamilyName: "Vendored") {
-                pattern(substring: "@notice")
-            }
-
-            // license types added by the project
-            for (Map.Entry<String, String[]> additional : additionalLicenses.entrySet()) {
-                String category = additional.getKey().substring(0, 5)
-                String family = additional.getKey().substring(5)
-                substringMatcher(licenseFamilyCategory: category,
-                        licenseFamilyName: family) {
-                    pattern(substring: additional.getValue())
-                }
-            }
-
-            // approved categories
-            for (String licenseFamily : approvedLicenses) {
-                approvedLicense(familyName: licenseFamily)
-            }
-        }
-
-        // check the license file for any errors, this should be fast.
-        boolean zeroUnknownLicenses = false
-        boolean foundProblemsWithFiles = false
-        reportFile.eachLine('UTF-8') { line ->
-            if (line.startsWith("0 Unknown Licenses")) {
-                zeroUnknownLicenses = true
-            }
-
-            if (line.startsWith(" !")) {
-                foundProblemsWithFiles = true
-            }
-        }
-
-        if (zeroUnknownLicenses == false || foundProblemsWithFiles) {
-            // print the unapproved license section, usually its all you need to fix problems.
-            int sectionNumber = 0
-            reportFile.eachLine('UTF-8') { line ->
-                if (line.startsWith("*******************************")) {
-                    sectionNumber++
-                } else {
-                    if (sectionNumber == 2) {
-                        logger.error(line)
-                    }
-                }
-            }
-            throw new IllegalStateException("License header problems were found! Full details: " + reportFile.absolutePath)
-        }
-    }
-
-    @Internal
-    abstract ListProperty<FileCollection> getSourceFolders();
-}

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy

@@ -1,102 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-package org.elasticsearch.gradle.test
-
-import groovy.transform.CompileStatic
-import org.elasticsearch.gradle.BuildPlugin
-import org.elasticsearch.gradle.ElasticsearchJavaPlugin
-import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask
-import org.elasticsearch.gradle.RepositoriesSetupPlugin
-import org.elasticsearch.gradle.info.BuildParams
-import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin
-import org.elasticsearch.gradle.internal.precommit.InternalPrecommitTasks
-import org.elasticsearch.gradle.precommit.PrecommitTasks
-import org.elasticsearch.gradle.testclusters.TestClustersPlugin
-import org.gradle.api.InvalidUserDataException
-import org.gradle.api.Plugin
-import org.gradle.api.Project
-import org.gradle.api.artifacts.Configuration
-import org.gradle.api.plugins.JavaBasePlugin
-import org.gradle.api.plugins.JavaPlugin
-import org.gradle.api.plugins.JavaPluginExtension
-import org.gradle.api.tasks.SourceSet
-import org.gradle.api.tasks.SourceSetContainer
-import org.gradle.api.tasks.testing.Test
-import org.gradle.plugins.ide.eclipse.model.EclipseModel
-import org.gradle.plugins.ide.idea.model.IdeaModel
-
-/**
- * Configures the build to compile tests against Elasticsearch's test framework
- * and run REST tests. Use BuildPlugin if you want to build main code as well
- * as tests.
- */
-@CompileStatic
-class StandaloneRestTestPlugin implements Plugin<Project> {
-
-    @Override
-    void apply(Project project) {
-        if (project.pluginManager.hasPlugin('elasticsearch.build')) {
-            throw new InvalidUserDataException('elasticsearch.standalone-test '
-                    + 'elasticsearch.standalone-rest-test, and elasticsearch.build '
-                    + 'are mutually exclusive')
-        }
-        project.rootProject.pluginManager.apply(GlobalBuildInfoPlugin)
-        project.pluginManager.apply(JavaBasePlugin)
-        project.pluginManager.apply(TestClustersPlugin)
-        project.pluginManager.apply(RepositoriesSetupPlugin)
-        project.pluginManager.apply(RestTestBasePlugin)
-
-        project.getTasks().register("buildResources", ExportElasticsearchBuildResourcesTask)
-        ElasticsearchJavaPlugin.configureInputNormalization(project)
-        ElasticsearchJavaPlugin.configureCompile(project)
-
-
-        project.extensions.getByType(JavaPluginExtension).sourceCompatibility = BuildParams.minimumRuntimeVersion
-        project.extensions.getByType(JavaPluginExtension).targetCompatibility = BuildParams.minimumRuntimeVersion
-
-        // only setup tests to build
-        SourceSetContainer sourceSets = project.extensions.getByType(SourceSetContainer)
-        SourceSet testSourceSet = sourceSets.create('test')
-
-        project.tasks.withType(Test).configureEach { Test test ->
-            test.testClassesDirs = testSourceSet.output.classesDirs
-            test.classpath = testSourceSet.runtimeClasspath
-        }
-
-        // create a compileOnly configuration as others might expect it
-        project.configurations.create("compileOnly")
-        project.dependencies.add('testImplementation', project.project(':test:framework'))
-
-        EclipseModel eclipse = project.extensions.getByType(EclipseModel)
-        eclipse.classpath.sourceSets = [testSourceSet]
-        eclipse.classpath.plusConfigurations = [project.configurations.getByName(JavaPlugin.TEST_RUNTIME_CLASSPATH_CONFIGURATION_NAME)]
-
-        IdeaModel idea = project.extensions.getByType(IdeaModel)
-        idea.module.testSourceDirs += testSourceSet.java.srcDirs
-        idea.module.scopes.put('TEST', [plus: [project.configurations.getByName(JavaPlugin.TEST_RUNTIME_CLASSPATH_CONFIGURATION_NAME)]] as Map<String, Collection<Configuration>>)
-
-        BuildParams.withInternalBuild {
-            InternalPrecommitTasks.create(project, false)
-        }.orElse {
-            PrecommitTasks.create(project)
-        }
-    }
-}

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestPlugin.groovy

@@ -1,49 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.gradle.test
-
-import groovy.transform.CompileStatic
-import org.elasticsearch.gradle.ElasticsearchJavaPlugin
-import org.gradle.api.Plugin
-import org.gradle.api.Project
-import org.gradle.api.plugins.JavaBasePlugin
-import org.gradle.api.tasks.testing.Test
-
-/**
- * Configures the build to compile against Elasticsearch's test framework and
- * run integration and unit tests. Use BuildPlugin if you want to build main
- * code as well as tests. */
-@CompileStatic
-class StandaloneTestPlugin implements Plugin<Project> {
-
-    @Override
-    void apply(Project project) {
-        project.pluginManager.apply(StandaloneRestTestPlugin)
-
-        project.tasks.register('test', Test).configure { t ->
-            t.group = JavaBasePlugin.VERIFICATION_GROUP
-            t.description = 'Runs unit tests that are separate'
-            t.mustRunAfter(project.tasks.getByName('precommit'))
-        }
-
-        ElasticsearchJavaPlugin.configureCompile(project)
-        project.tasks.named('check').configure { it.dependsOn(project.tasks.named('test')) }
-    }
-}

buildSrc/src/main/java/org/elasticsearch/gradle/BuildPlugin.java

@@ -0,0 +1,99 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle;
+
+import org.codehaus.groovy.runtime.DefaultGroovyMethods;
+import org.elasticsearch.gradle.info.BuildParams;
+import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin;
+import org.elasticsearch.gradle.internal.InternalPlugin;
+import org.elasticsearch.gradle.internal.precommit.InternalPrecommitTasks;
+import org.elasticsearch.gradle.precommit.PrecommitTasks;
+import org.gradle.api.GradleException;
+import org.gradle.api.InvalidUserDataException;
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.plugins.ExtraPropertiesExtension;
+import org.gradle.api.tasks.bundling.Jar;
+
+import java.io.File;
+
+/**
+ * Encapsulates build configuration for elasticsearch projects.
+ */
+public class BuildPlugin implements Plugin<Project> {
+    @Override
+    public void apply(final Project project) {
+        // make sure the global build info plugin is applied to the root project
+        project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class);
+        checkExternalInternalPluginUsages(project);
+
+        if (project.getPluginManager().hasPlugin("elasticsearch.standalone-rest-test")) {
+            throw new InvalidUserDataException(
+                "elasticsearch.standalone-test, " + "elasticsearch.standalone-rest-test, and elasticsearch.build are mutually exclusive"
+            );
+        }
+
+        project.getPluginManager().apply("elasticsearch.java");
+        configureLicenseAndNotice(project);
+        project.getPluginManager().apply("elasticsearch.publish");
+        project.getPluginManager().apply(DependenciesInfoPlugin.class);
+        project.getPluginManager().apply(DependenciesGraphPlugin.class);
+
+        BuildParams.withInternalBuild(() -> InternalPrecommitTasks.create(project, true)).orElse(() -> PrecommitTasks.create(project));
+
+    }
+
+    private static void checkExternalInternalPluginUsages(Project project) {
+        if (BuildParams.isInternal().equals(false)) {
+            project.getPlugins()
+                .withType(
+                    InternalPlugin.class,
+                    internalPlugin -> { throw new GradleException(internalPlugin.getExternalUseErrorMessage()); }
+                );
+        }
+    }
+
+    public static void configureLicenseAndNotice(final Project project) {
+        final ExtraPropertiesExtension ext = project.getExtensions().getByType(ExtraPropertiesExtension.class);
+        ext.set("licenseFile", null);
+        ext.set("noticeFile", null);
+        // add license/notice files
+        project.afterEvaluate(p -> p.getTasks().withType(Jar.class).configureEach(jar -> {
+            if (ext.has("licenseFile") == false
+                || ext.get("licenseFile") == null
+                || ext.has("noticeFile") == false
+                || ext.get("noticeFile") == null) {
+                throw new GradleException("Must specify license and notice file for project " + p.getPath());
+            }
+            final File licenseFile = DefaultGroovyMethods.asType(ext.get("licenseFile"), File.class);
+            final File noticeFile = DefaultGroovyMethods.asType(ext.get("noticeFile"), File.class);
+            jar.metaInf(spec -> {
+                spec.from(licenseFile.getParent(), from -> {
+                    from.include(licenseFile.getName());
+                    from.rename(s -> "LICENSE.txt");
+                });
+                spec.from(noticeFile.getParent(), from -> {
+                    from.include(noticeFile.getName());
+                    from.rename(s -> "NOTICE.txt");
+                });
+            });
+        }));
+    }
+}

buildSrc/src/main/groovy/org/elasticsearch/gradle/DependenciesInfoPlugin.groovy → buildSrc/src/main/java/org/elasticsearch/gradle/DependenciesInfoPlugin.java

@@ -17,27 +17,29 @@
  * under the License.
  */
 
-package org.elasticsearch.gradle
+package org.elasticsearch.gradle;
 
-import org.elasticsearch.gradle.dependencies.CompileOnlyResolvePlugin
-import org.elasticsearch.gradle.internal.precommit.DependencyLicensesTask
-import org.gradle.api.Plugin
-import org.gradle.api.Project
-import org.gradle.api.plugins.JavaPlugin
-import org.gradle.api.tasks.TaskProvider
+import org.elasticsearch.gradle.dependencies.CompileOnlyResolvePlugin;
+import org.elasticsearch.gradle.internal.precommit.DependencyLicensesTask;
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.plugins.JavaPlugin;
 
-class DependenciesInfoPlugin implements Plugin<Project> {
+public class DependenciesInfoPlugin implements Plugin<Project> {
     @Override
-    void apply(Project project) {
+    public void apply(final Project project) {
         project.getPlugins().apply(CompileOnlyResolvePlugin.class);
-        TaskProvider<DependenciesInfoTask> depsInfo = project.getTasks().register("dependenciesInfo", DependenciesInfoTask.class);
-        depsInfo.configure { DependenciesInfoTask t ->
+        var depsInfo = project.getTasks().register("dependenciesInfo", DependenciesInfoTask.class);
+        depsInfo.configure(t -> {
             t.setRuntimeConfiguration(project.getConfigurations().getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME));
-            t.setCompileOnlyConfiguration(project.getConfigurations().getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME));
-            t.getConventionMapping().map("mappings") { ->
-                TaskProvider<DependencyLicensesTask> depLic = project.getTasks().named("dependencyLicenses", DependencyLicensesTask.class);
+            t.setCompileOnlyConfiguration(
+                project.getConfigurations().getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME)
+            );
+            t.getConventionMapping().map("mappings", () -> {
+                var depLic = project.getTasks().named("dependencyLicenses", DependencyLicensesTask.class);
                 return depLic.get().getMappings();
-            }
-        }
+            });
+        });
     }
+
 }

buildSrc/src/main/java/org/elasticsearch/gradle/DependenciesInfoTask.java

@@ -0,0 +1,239 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle;
+
+import org.elasticsearch.gradle.internal.precommit.DependencyLicensesTask;
+import org.elasticsearch.gradle.precommit.LicenseAnalyzer;
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.artifacts.Dependency;
+import org.gradle.api.artifacts.DependencySet;
+import org.gradle.api.artifacts.ModuleVersionIdentifier;
+import org.gradle.api.artifacts.ProjectDependency;
+import org.gradle.api.internal.ConventionTask;
+import org.gradle.api.tasks.Input;
+import org.gradle.api.tasks.InputDirectory;
+import org.gradle.api.tasks.InputFiles;
+import org.gradle.api.tasks.Optional;
+import org.gradle.api.tasks.OutputFile;
+import org.gradle.api.tasks.TaskAction;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.StandardOpenOption;
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.Set;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+/**
+ * A task to gather information about the dependencies and export them into a csv file.
+ * <p>
+ * The following information is gathered:
+ * <ul>
+ *     <li>name: name that identifies the library (groupId:artifactId)</li>
+ *     <li>version</li>
+ *     <li>URL: link to have more information about the dependency.</li>
+ *     <li>license: <a href="https://spdx.org/licenses/">SPDX license</a> identifier, custom license or UNKNOWN.</li>
+ * </ul>
+ */
+public class DependenciesInfoTask extends ConventionTask {
+    /**
+     * Directory to read license files
+     */
+    @Optional
+    @InputDirectory
+    private File licensesDir = new File(getProject().getProjectDir(), "licenses").exists()
+        ? new File(getProject().getProjectDir(), "licenses")
+        : null;
+
+    @OutputFile
+    private File outputFile = new File(getProject().getBuildDir(), "reports/dependencies/dependencies.csv");
+    private LinkedHashMap<String, String> mappings;
+
+    public Configuration getRuntimeConfiguration() {
+        return runtimeConfiguration;
+    }
+
+    public void setRuntimeConfiguration(Configuration runtimeConfiguration) {
+        this.runtimeConfiguration = runtimeConfiguration;
+    }
+
+    public Configuration getCompileOnlyConfiguration() {
+        return compileOnlyConfiguration;
+    }
+
+    public void setCompileOnlyConfiguration(Configuration compileOnlyConfiguration) {
+        this.compileOnlyConfiguration = compileOnlyConfiguration;
+    }
+
+    public File getLicensesDir() {
+        return licensesDir;
+    }
+
+    public void setLicensesDir(File licensesDir) {
+        this.licensesDir = licensesDir;
+    }
+
+    public File getOutputFile() {
+        return outputFile;
+    }
+
+    public void setOutputFile(File outputFile) {
+        this.outputFile = outputFile;
+    }
+
+    /**
+     * Dependencies to gather information from.
+     */
+    @InputFiles
+    private Configuration runtimeConfiguration;
+    /**
+     * We subtract compile-only dependencies.
+     */
+    @InputFiles
+    private Configuration compileOnlyConfiguration;
+
+    public DependenciesInfoTask() {
+        setDescription("Create a CSV file with dependencies information.");
+    }
+
+    @TaskAction
+    public void generateDependenciesInfo() throws IOException {
+
+        final DependencySet runtimeDependencies = runtimeConfiguration.getAllDependencies();
+        // we have to resolve the transitive dependencies and create a group:artifactId:version map
+
+        final Set<String> compileOnlyArtifacts = compileOnlyConfiguration.getResolvedConfiguration()
+            .getResolvedArtifacts()
+            .stream()
+            .map(r -> {
+                ModuleVersionIdentifier id = r.getModuleVersion().getId();
+                return id.getGroup() + ":" + id.getName() + ":" + id.getVersion();
+            })
+            .collect(Collectors.toSet());
+
+        final StringBuilder output = new StringBuilder();
+        for (final Dependency dep : runtimeDependencies) {
+            // we do not need compile-only dependencies here
+            if (compileOnlyArtifacts.contains(dep.getGroup() + ":" + dep.getName() + ":" + dep.getVersion())) {
+                continue;
+            }
+
+            // only external dependencies are checked
+            if (dep instanceof ProjectDependency) {
+                continue;
+            }
+
+            final String url = createURL(dep.getGroup(), dep.getName(), dep.getVersion());
+            final String dependencyName = DependencyLicensesTask.getDependencyName(getMappings(), dep.getName());
+            getLogger().info("mapped dependency " + dep.getGroup() + ":" + dep.getName() + " to " + dependencyName + " for license info");
+
+            final String licenseType = getLicenseType(dep.getGroup(), dependencyName);
+            output.append(dep.getGroup() + ":" + dep.getName() + "," + dep.getVersion() + "," + url + "," + licenseType + "\n");
+        }
+
+        Files.write(outputFile.toPath(), output.toString().getBytes("UTF-8"), StandardOpenOption.CREATE);
+    }
+
+    @Input
+    public LinkedHashMap<String, String> getMappings() {
+        return mappings;
+    }
+
+    public void setMappings(LinkedHashMap<String, String> mappings) {
+        this.mappings = mappings;
+    }
+
+    /**
+     * Create an URL on <a href="https://repo1.maven.org/maven2/">Maven Central</a>
+     * based on dependency coordinates.
+     */
+    protected String createURL(final String group, final String name, final String version) {
+        final String baseURL = "https://repo1.maven.org/maven2";
+        return baseURL + "/" + group.replaceAll("\\.", "/") + "/" + name + "/" + version;
+    }
+
+    /**
+     * Read the LICENSE file associated with the dependency and determine a license type.
+     * <p>
+     * The license type is one of the following values:
+     * <ul>
+     * <li><em>UNKNOWN</em> if LICENSE file is not present for this dependency.</li>
+     * <li><em>one SPDX identifier</em> if the LICENSE content matches with an SPDX license.</li>
+     * <li><em>Custom;URL</em> if it's not an SPDX license,
+     * URL is the Github URL to the LICENSE file in elasticsearch repository.</li>
+     * </ul>
+     *
+     * @param group dependency group
+     * @param name  dependency name
+     * @return SPDX identifier, UNKNOWN or a Custom license
+     */
+    protected String getLicenseType(final String group, final String name) throws IOException {
+        final File license = getDependencyInfoFile(group, name, "LICENSE");
+        String licenseType;
+
+        final LicenseAnalyzer.LicenseInfo licenseInfo = LicenseAnalyzer.licenseType(license);
+        if (licenseInfo.isSpdxLicense() == false) {
+            // License has not be identified as SPDX.
+            // As we have the license file, we create a Custom entry with the URL to this license file.
+            final String gitBranch = System.getProperty("build.branch", "master");
+            final String githubBaseURL = "https://raw.githubusercontent.com/elastic/elasticsearch/" + gitBranch + "/";
+            licenseType = licenseInfo.getIdentifier()
+                + ";"
+                + license.getCanonicalPath().replaceFirst(".*/elasticsearch/", githubBaseURL)
+                + ",";
+        } else {
+            licenseType = licenseInfo.getIdentifier() + ",";
+        }
+
+        if (licenseInfo.isSourceRedistributionRequired()) {
+            final File sources = getDependencyInfoFile(group, name, "SOURCES");
+            licenseType += Files.readString(sources.toPath()).trim();
+        }
+
+        return licenseType;
+    }
+
+    protected File getDependencyInfoFile(final String group, final String name, final String infoFileSuffix) {
+        java.util.Optional<File> license = licensesDir != null
+            ? Arrays.stream(licensesDir.listFiles((dir, name1) -> Pattern.matches("/" + ".*-" + infoFileSuffix + ".*" + "/", name1)))
+                .filter(file -> {
+                    String prefix = file.getName().split("-" + infoFileSuffix + ".*")[0];
+                    return group.contains(prefix) || name.contains(prefix);
+                })
+                .findFirst()
+            : java.util.Optional.empty();
+
+        return license.orElseThrow(
+            () -> new IllegalStateException(
+                "Unable to find "
+                    + infoFileSuffix
+                    + " file for dependency "
+                    + group
+                    + ":"
+                    + name
+                    + " in "
+                    + getLicensesDir().getAbsolutePath()
+            )
+        );
+    }
+}

buildSrc/src/main/groovy/org/elasticsearch/gradle/internal/precommit/InternalPrecommitTasks.groovy → buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/InternalPrecommitTasks.java

@@ -17,25 +17,21 @@
  * under the License.
  */
 
-package org.elasticsearch.gradle.internal.precommit
+package org.elasticsearch.gradle.internal.precommit;
 
-import groovy.transform.CompileStatic
-import org.elasticsearch.gradle.precommit.PrecommitTasks
+import org.elasticsearch.gradle.precommit.PrecommitTasks;
 import org.elasticsearch.gradle.precommit.ThirdPartyAuditPrecommitPlugin;
 import org.gradle.api.Project;
 
 /**
  * Internal precommit plugins that adds elasticsearch project specific
  * checks to the common precommit plugin.
- * */
-
-@CompileStatic
-class InternalPrecommitTasks {
-
+ */
+public class InternalPrecommitTasks {
     /**
      * Adds a precommit task, which depends on non-test verification tasks.
      */
-    static void create(Project project, boolean includeDependencyLicenses) {
+    public static void create(Project project, boolean includeDependencyLicenses) {
         PrecommitTasks.create(project);
 
         project.getPluginManager().apply(ThirdPartyAuditPrecommitPlugin.class);
@@ -66,5 +62,4 @@ class InternalPrecommitTasks {
             project.getPluginManager().apply(LoggerUsagePrecommitPlugin.class);
         }
     }
-
 }

buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/LicenseHeadersPrecommitPlugin.java

@@ -0,0 +1,53 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle.internal.precommit;
+
+import org.elasticsearch.gradle.internal.InternalPlugin;
+import org.elasticsearch.gradle.precommit.PrecommitPlugin;
+import org.elasticsearch.gradle.util.GradleUtils;
+import org.gradle.api.Project;
+import org.gradle.api.Task;
+import org.gradle.api.plugins.JavaBasePlugin;
+import org.gradle.api.provider.ProviderFactory;
+import org.gradle.api.tasks.SourceSetContainer;
+import org.gradle.api.tasks.TaskProvider;
+
+import javax.inject.Inject;
+import java.util.stream.Collectors;
+
+public class LicenseHeadersPrecommitPlugin extends PrecommitPlugin implements InternalPlugin {
+    @Inject
+    public LicenseHeadersPrecommitPlugin(ProviderFactory providerFactory) {
+        this.providerFactory = providerFactory;
+    }
+
+    @Override
+    public TaskProvider<? extends Task> createTask(Project project) {
+        return project.getTasks().register("licenseHeaders", LicenseHeadersTask.class, licenseHeadersTask -> {
+            project.getPlugins().withType(JavaBasePlugin.class, javaBasePlugin -> {
+                final SourceSetContainer sourceSets = GradleUtils.getJavaSourceSets(project);
+                licenseHeadersTask.getSourceFolders()
+                    .addAll(providerFactory.provider(() -> sourceSets.stream().map(s -> s.getAllJava()).collect(Collectors.toList())));
+            });
+        });
+    }
+
+    private ProviderFactory providerFactory;
+}

buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/LicenseHeadersTask.java

@@ -0,0 +1,258 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle.internal.precommit;
+
+import org.apache.rat.Defaults;
+import org.apache.rat.ReportConfiguration;
+import org.apache.rat.analysis.IHeaderMatcher;
+import org.apache.rat.analysis.util.HeaderMatcherMultiplexer;
+import org.apache.rat.anttasks.SubstringLicenseMatcher;
+import org.apache.rat.api.RatException;
+import org.apache.rat.document.impl.FileDocument;
+import org.apache.rat.license.SimpleLicenseFamily;
+import org.apache.rat.report.RatReport;
+import org.apache.rat.report.claim.ClaimStatistic;
+import org.apache.rat.report.xml.XmlReportFactory;
+import org.apache.rat.report.xml.writer.impl.base.XmlWriter;
+import org.gradle.api.DefaultTask;
+import org.gradle.api.GradleException;
+import org.gradle.api.file.FileCollection;
+import org.gradle.api.provider.ListProperty;
+import org.gradle.api.tasks.CacheableTask;
+import org.gradle.api.tasks.Input;
+import org.gradle.api.tasks.InputFiles;
+import org.gradle.api.tasks.Internal;
+import org.gradle.api.tasks.OutputFile;
+import org.gradle.api.tasks.PathSensitive;
+import org.gradle.api.tasks.PathSensitivity;
+import org.gradle.api.tasks.SkipWhenEmpty;
+import org.gradle.api.tasks.TaskAction;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
+
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.Writer;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * Checks files for license headers..
+ */
+@CacheableTask
+public abstract class LicenseHeadersTask extends DefaultTask {
+    public LicenseHeadersTask() {
+        setDescription("Checks sources for missing, incorrect, or unacceptable license headers");
+    }
+
+    /**
+     * The list of java files to check. protected so the afterEvaluate closure in the
+     * constructor can write to it.
+     */
+    @InputFiles
+    @SkipWhenEmpty
+    @PathSensitive(PathSensitivity.RELATIVE)
+    public List<FileCollection> getJavaFiles() {
+        return getSourceFolders().get();
+    }
+
+    @Internal
+    public abstract ListProperty<FileCollection> getSourceFolders();
+
+    public File getReportFile() {
+        return reportFile;
+    }
+
+    public void setReportFile(File reportFile) {
+        this.reportFile = reportFile;
+    }
+
+    public List<String> getApprovedLicenses() {
+        return approvedLicenses;
+    }
+
+    public void setApprovedLicenses(List<String> approvedLicenses) {
+        this.approvedLicenses = approvedLicenses;
+    }
+
+    public List<String> getExcludes() {
+        return excludes;
+    }
+
+    public Map<String, String> getAdditionalLicenses() {
+        return additionalLicenses;
+    }
+
+    public void setExcludes(List<String> excludes) {
+        this.excludes = excludes;
+    }
+
+    @OutputFile
+    private File reportFile = new File(getProject().getBuildDir(), "reports/licenseHeaders/rat.xml");
+
+    /**
+     * Allowed license families for this project.
+     */
+    @Input
+    private List<String> approvedLicenses = new ArrayList<String>(Arrays.asList("Apache", "Generated", "Vendored"));
+    /**
+     * Files that should be excluded from the license header check. Use with extreme care, only in situations where the license on the
+     * source file is compatible with the codebase but we do not want to add the license to the list of approved headers (to avoid the
+     * possibility of inadvertently using the license on our own source files).
+     */
+    @Input
+    private List<String> excludes = new ArrayList<String>();
+    /**
+     * Additional license families that may be found. The key is the license category name (5 characters),
+     * followed by the family name and the value list of patterns to search for.
+     */
+    @Input
+    protected Map<String, String> additionalLicenses = new HashMap<String, String>();
+
+    /**
+     * Add a new license type.
+     * <p>
+     * The license may be added to the {@link #approvedLicenses} using the {@code familyName}.
+     *
+     * @param categoryName A 5-character string identifier for the license
+     * @param familyName   An expanded string name for the license
+     * @param pattern      A pattern to search for, which if found, indicates a file contains the license
+     */
+    public void additionalLicense(final String categoryName, String familyName, String pattern) {
+        if (categoryName.length() != 5) {
+            throw new IllegalArgumentException("License category name must be exactly 5 characters, got " + categoryName);
+        }
+
+        additionalLicenses.put(categoryName + familyName, pattern);
+    }
+
+    @TaskAction
+    public void runRat() {
+        ReportConfiguration reportConfiguration = new ReportConfiguration();
+        reportConfiguration.setAddingLicenses(true);
+        List<IHeaderMatcher> matchers = new ArrayList<>();
+        matchers.add(Defaults.createDefaultMatcher());
+
+        // BSD 4-clause stuff (is disallowed below)
+        // we keep this here, in case someone adds BSD code for some reason, it should never be allowed.
+        matchers.add(subStringMatcher("BSD4 ", "Original BSD License (with advertising clause)", "All advertising materials"));
+        // Apache
+        matchers.add(subStringMatcher("AL   ", "Apache", "Licensed to Elasticsearch under one or more contributor"));
+        // Generated resources
+        matchers.add(subStringMatcher("GEN  ", "Generated", "ANTLR GENERATED CODE"));
+        // Vendored Code
+        matchers.add(subStringMatcher("VEN  ", "Vendored", "@notice"));
+
+        for (Map.Entry<String, String> additional : additionalLicenses.entrySet()) {
+            String category = additional.getKey().substring(0, 5);
+            String family = additional.getKey().substring(5);
+            matchers.add(subStringMatcher(category, family, additional.getValue()));
+        }
+
+        reportConfiguration.setHeaderMatcher(new HeaderMatcherMultiplexer(matchers.toArray(IHeaderMatcher[]::new)));
+        reportConfiguration.setApprovedLicenseNames(approvedLicenses.stream().map(license -> {
+            SimpleLicenseFamily simpleLicenseFamily = new SimpleLicenseFamily();
+            simpleLicenseFamily.setFamilyName(license);
+            return simpleLicenseFamily;
+        }).toArray(SimpleLicenseFamily[]::new));
+
+        ClaimStatistic stats = generateReport(reportConfiguration, getReportFile());
+        boolean unknownLicenses = stats.getNumUnknown() > 0;
+        boolean unApprovedLicenses = stats.getNumUnApproved() > 0;
+        if (unknownLicenses || unApprovedLicenses) {
+            unapprovedFiles(getReportFile()).stream().forEachOrdered(unapprovedFile -> getLogger().error(unapprovedFile));
+            throw new GradleException("License header problems were found! Full details: " + reportFile.getAbsolutePath());
+        }
+    }
+
+    private IHeaderMatcher subStringMatcher(String licenseFamilyCategory, String licenseFamilyName, String substringPattern) {
+        SubstringLicenseMatcher substringLicenseMatcher = new SubstringLicenseMatcher();
+        substringLicenseMatcher.setLicenseFamilyCategory(licenseFamilyCategory);
+        substringLicenseMatcher.setLicenseFamilyName(licenseFamilyName);
+
+        SubstringLicenseMatcher.Pattern pattern = new SubstringLicenseMatcher.Pattern();
+        pattern.setSubstring(substringPattern);
+        substringLicenseMatcher.addConfiguredPattern(pattern);
+        return substringLicenseMatcher;
+    }
+
+    private ClaimStatistic generateReport(ReportConfiguration config, File xmlReportFile) {
+        try {
+            Files.deleteIfExists(reportFile.toPath());
+            BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter(xmlReportFile));
+            return toXmlReportFile(config, bufferedWriter);
+        } catch (IOException | RatException exception) {
+            throw new GradleException("Cannot generate license header report for " + getPath(), exception);
+        }
+    }
+
+    private ClaimStatistic toXmlReportFile(ReportConfiguration config, Writer writer) throws RatException, IOException {
+        ClaimStatistic stats = new ClaimStatistic();
+        RatReport standardReport = XmlReportFactory.createStandardReport(new XmlWriter(writer), stats, config);
+
+        standardReport.startReport();
+        for (FileCollection dirSet : getSourceFolders().get()) {
+            for (File f : dirSet.getAsFileTree().matching(patternFilterable -> patternFilterable.exclude(getExcludes()))) {
+                standardReport.report(new FileDocument(f));
+            }
+        }
+        standardReport.endReport();
+        writer.flush();
+        writer.close();
+        return stats;
+    }
+
+    private static List<String> unapprovedFiles(File xmlReportFile) {
+        try {
+            NodeList resourcesNodes = DocumentBuilderFactory.newInstance()
+                .newDocumentBuilder()
+                .parse(xmlReportFile)
+                .getElementsByTagName("resource");
+            return elementList(resourcesNodes).stream()
+                .filter(
+                    resource -> elementList(resource.getChildNodes()).stream()
+                        .anyMatch(n -> n.getTagName().equals("license-approval") && n.getAttribute("name").equals("false"))
+                )
+                .map(e -> e.getAttribute("name"))
+                .sorted()
+                .collect(Collectors.toList());
+        } catch (SAXException | IOException | ParserConfigurationException e) {
+            throw new GradleException("Error parsing xml report " + xmlReportFile.getAbsolutePath());
+        }
+    }
+
+    private static List<Element> elementList(NodeList resourcesNodes) {
+        List<Element> nodeList = new ArrayList<>(resourcesNodes.getLength());
+        for (int idx = 0; idx < resourcesNodes.getLength(); idx++) {
+            nodeList.add((Element) resourcesNodes.item(idx));
+        }
+        return nodeList;
+    }
+}

buildSrc/src/main/java/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.java

@@ -0,0 +1,100 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle.test;
+
+import org.elasticsearch.gradle.ElasticsearchJavaPlugin;
+import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask;
+import org.elasticsearch.gradle.RepositoriesSetupPlugin;
+import org.elasticsearch.gradle.info.BuildParams;
+import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin;
+import org.elasticsearch.gradle.internal.precommit.InternalPrecommitTasks;
+import org.elasticsearch.gradle.precommit.PrecommitTasks;
+import org.elasticsearch.gradle.testclusters.TestClustersPlugin;
+import org.gradle.api.InvalidUserDataException;
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.plugins.JavaBasePlugin;
+import org.gradle.api.plugins.JavaPlugin;
+import org.gradle.api.plugins.JavaPluginExtension;
+import org.gradle.api.tasks.SourceSet;
+import org.gradle.api.tasks.SourceSetContainer;
+import org.gradle.api.tasks.testing.Test;
+import org.gradle.plugins.ide.eclipse.model.EclipseModel;
+
+import org.gradle.plugins.ide.idea.model.IdeaModel;
+
+import java.util.Arrays;
+import java.util.Map;
+
+/**
+ * Configures the build to compile tests against Elasticsearch's test framework
+ * and run REST tests. Use BuildPlugin if you want to build main code as well
+ * as tests.
+ */
+public class StandaloneRestTestPlugin implements Plugin<Project> {
+    @Override
+    public void apply(final Project project) {
+        if (project.getPluginManager().hasPlugin("elasticsearch.build")) {
+            throw new InvalidUserDataException(
+                "elasticsearch.standalone-test, elasticsearch.standalone-rest-test, " + "and elasticsearch.build are mutually exclusive"
+            );
+        }
+
+        project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class);
+        project.getPluginManager().apply(JavaBasePlugin.class);
+        project.getPluginManager().apply(TestClustersPlugin.class);
+        project.getPluginManager().apply(RepositoriesSetupPlugin.class);
+        project.getPluginManager().apply(RestTestBasePlugin.class);
+
+        project.getTasks().register("buildResources", ExportElasticsearchBuildResourcesTask.class);
+        ElasticsearchJavaPlugin.configureInputNormalization(project);
+        ElasticsearchJavaPlugin.configureCompile(project);
+
+        project.getExtensions().getByType(JavaPluginExtension.class).setSourceCompatibility(BuildParams.getMinimumRuntimeVersion());
+        project.getExtensions().getByType(JavaPluginExtension.class).setTargetCompatibility(BuildParams.getMinimumRuntimeVersion());
+
+        // only setup tests to build
+        SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
+        final SourceSet testSourceSet = sourceSets.create("test");
+
+        project.getTasks().withType(Test.class).configureEach(test -> test.setTestClassesDirs(testSourceSet.getOutput().getClassesDirs()));
+
+        // create a compileOnly configuration as others might expect it
+        project.getConfigurations().create("compileOnly");
+        project.getDependencies().add("testImplementation", project.project(":test:framework"));
+
+        EclipseModel eclipse = project.getExtensions().getByType(EclipseModel.class);
+        eclipse.getClasspath().setSourceSets(Arrays.asList(testSourceSet));
+        eclipse.getClasspath()
+            .setPlusConfigurations(
+                Arrays.asList(project.getConfigurations().getByName(JavaPlugin.TEST_RUNTIME_CLASSPATH_CONFIGURATION_NAME))
+            );
+
+        IdeaModel idea = project.getExtensions().getByType(IdeaModel.class);
+        idea.getModule().getTestSourceDirs().addAll(testSourceSet.getJava().getSrcDirs());
+        idea.getModule()
+            .getScopes()
+            .put(
+                "TEST",
+                Map.of("plus", Arrays.asList(project.getConfigurations().getByName(JavaPlugin.TEST_RUNTIME_CLASSPATH_CONFIGURATION_NAME)))
+            );
+        BuildParams.withInternalBuild(() -> InternalPrecommitTasks.create(project, false)).orElse(() -> PrecommitTasks.create(project));
+    }
+}

buildSrc/src/main/java/org/elasticsearch/gradle/test/StandaloneTestPlugin.java

@@ -0,0 +1,47 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle.test;
+
+import org.elasticsearch.gradle.ElasticsearchJavaPlugin;
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.plugins.JavaBasePlugin;
+import org.gradle.api.tasks.testing.Test;
+
+/**
+ * Configures the build to compile against Elasticsearch's test framework and
+ * run integration and unit tests. Use BuildPlugin if you want to build main
+ * code as well as tests.
+ */
+public class StandaloneTestPlugin implements Plugin<Project> {
+    @Override
+    public void apply(final Project project) {
+        project.getPluginManager().apply(StandaloneRestTestPlugin.class);
+
+        project.getTasks().register("test", Test.class).configure(test -> {
+            test.setGroup(JavaBasePlugin.VERIFICATION_GROUP);
+            test.setDescription("Runs unit tests that are separate");
+            test.mustRunAfter(project.getTasks().getByName("precommit"));
+        });
+
+        ElasticsearchJavaPlugin.configureCompile(project);
+        project.getTasks().named("check").configure(task -> task.dependsOn(project.getTasks().named("test")));
+    }
+}

buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.internal-licenseheaders.properties

@@ -0,0 +1,21 @@
+#
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# needed for testkit testing
+implementation-class=org.elasticsearch.gradle.internal.precommit.LicenseHeadersPrecommitPlugin