Run Third party audit with forbidden APIs CLI (part3/3) (#33052)

The new implementation is functional equivalent with the old, ant based one.
It parses task standard error to get the missing classes and violations in the same way.
I considered re-using ForbiddenApisCliTask but Gradle makes it hard to build inheritance with tasks that have task actions , since the order of the task actions can't be controlled.
This inheritance isn't dully desired either as the third party audit task is much more opinionated and we don't want to expose some of the configuration.
We could probably extract a common base class without any task actions, but probably more trouble than it's worth.

Closes #31715

buildSrc/build.gradle

@@ -102,7 +102,6 @@ dependencies {
   compile 'com.netflix.nebula:gradle-info-plugin:3.0.3'
   compile 'org.eclipse.jgit:org.eclipse.jgit:3.2.0.201312181205-r'
   compile 'com.perforce:p4java:2012.3.551082' // THIS IS SUPPOSED TO BE OPTIONAL IN THE FUTURE....
-  compile 'de.thetaphi:forbiddenapis:2.5'
   compile 'org.apache.rat:apache-rat:0.11'
   compile "org.elasticsearch:jna:4.5.1"
   compile 'com.github.jengelman.gradle.plugins:shadow:2.0.4'

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy

@@ -31,6 +31,11 @@ class PrecommitTasks {
 
     /** Adds a precommit task, which depends on non-test verification tasks. */
     public static Task create(Project project, boolean includeDependencyLicenses) {
+        Configuration forbiddenApisConfiguration = project.configurations.create("forbiddenApisCliJar")
+        project.dependencies {
+            forbiddenApisCliJar ('de.thetaphi:forbiddenapis:2.5')
+        }
+
         List<Task> precommitTasks = [
             configureCheckstyle(project),
             configureForbiddenApisCli(project),
@@ -39,7 +44,7 @@ class PrecommitTasks {
             project.tasks.create('licenseHeaders', LicenseHeadersTask.class),
             project.tasks.create('filepermissions', FilePermissionsTask.class),
             project.tasks.create('jarHell', JarHellTask.class),
-            project.tasks.create('thirdPartyAudit', ThirdPartyAuditTask.class)
+            configureThirdPartyAudit(project)
         ]
 
         // tasks with just tests don't need dependency licenses, so this flag makes adding
@@ -75,32 +80,26 @@ class PrecommitTasks {
         return project.tasks.create(precommitOptions)
     }
 
-    private static Task configureForbiddenApisCli(Project project) {
-        Configuration forbiddenApisConfiguration = project.configurations.create("forbiddenApisCliJar")
-        project.dependencies {
-            forbiddenApisCliJar ('de.thetaphi:forbiddenapis:2.5')
+    private static Task configureThirdPartyAudit(Project project) {
+        ThirdPartyAuditTask thirdPartyAuditTask = project.tasks.create('thirdPartyAudit', ThirdPartyAuditTask.class)
+        ExportElasticsearchBuildResourcesTask buildResources = project.tasks.getByName('buildResources')
+        thirdPartyAuditTask.configure {
+            dependsOn(buildResources)
+            signatureFile = buildResources.copy("forbidden/third-party-audit.txt")
+            javaHome = project.runtimeJavaHome
         }
-        Task forbiddenApisCli = project.tasks.create('forbiddenApis')
+        return thirdPartyAuditTask
+    }
 
+    private static Task configureForbiddenApisCli(Project project) {
+        Task forbiddenApisCli = project.tasks.create('forbiddenApis')
         project.sourceSets.forEach { sourceSet ->
             forbiddenApisCli.dependsOn(
                 project.tasks.create(sourceSet.getTaskName('forbiddenApis', null), ForbiddenApisCliTask) {
                     ExportElasticsearchBuildResourcesTask buildResources = project.tasks.getByName('buildResources')
                     dependsOn(buildResources)
-                    execAction = { spec ->
-                        spec.classpath = project.files(
-                                project.configurations.forbiddenApisCliJar,
-                                sourceSet.compileClasspath,
-                                sourceSet.runtimeClasspath
-                        )
-                        spec.executable = "${project.runtimeJavaHome}/bin/java"
-                    }
-                    inputs.files(
-                            forbiddenApisConfiguration,
-                            sourceSet.compileClasspath,
-                            sourceSet.runtimeClasspath
-                    )
-
+                    it.sourceSet = sourceSet
+                    javaHome = project.runtimeJavaHome
                     targetCompatibility = project.compilerJavaVersion
                     bundledSignatures = [
                        "jdk-unsafe", "jdk-deprecated", "jdk-non-portable", "jdk-system-out"

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.groovy

@@ -1,297 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.gradle.precommit;
-
-import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin
-import org.apache.tools.ant.BuildEvent;
-import org.apache.tools.ant.BuildException;
-import org.apache.tools.ant.BuildListener;
-import org.apache.tools.ant.BuildLogger;
-import org.apache.tools.ant.DefaultLogger;
-import org.apache.tools.ant.Project;
-import org.elasticsearch.gradle.AntTask;
-import org.gradle.api.artifacts.Configuration;
-import org.gradle.api.file.FileCollection;
-import org.gradle.api.tasks.Input
-import org.gradle.api.tasks.InputFiles
-import org.gradle.api.tasks.OutputFile
-
-import java.nio.file.FileVisitResult;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.SimpleFileVisitor;
-import java.nio.file.attribute.BasicFileAttributes;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * Basic static checking to keep tabs on third party JARs
- */
-public class ThirdPartyAuditTask extends AntTask {
-
-    // patterns for classes to exclude, because we understand their issues
-    private List<String> excludes = [];
-
-    /**
-     * Input for the task. Set javadoc for {#link getJars} for more. Protected
-     * so the afterEvaluate closure in the constructor can write it.
-     */
-    protected FileCollection jars;
-
-    /**
-     * Classpath against which to run the third patty audit. Protected so the
-     * afterEvaluate closure in the constructor can write it.
-     */
-    protected FileCollection classpath;
-
-    /**
-     * We use a simple "marker" file that we touch when the task succeeds
-     * as the task output. This is compared against the modified time of the
-     * inputs (ie the jars/class files).
-     */
-    @OutputFile
-    File successMarker = new File(project.buildDir, 'markers/thirdPartyAudit')
-
-    ThirdPartyAuditTask() {
-        // we depend on this because its the only reliable configuration
-        // this probably makes the build slower: gradle you suck here when it comes to configurations, you pay the price.
-        dependsOn(project.configurations.testCompile);
-        description = "Checks third party JAR bytecode for missing classes, use of internal APIs, and other horrors'";
-
-        project.afterEvaluate {
-            Configuration configuration = project.configurations.findByName('runtime')
-            Configuration compileOnly = project.configurations.findByName('compileOnly')
-            if (configuration == null) {
-                // some projects apparently do not have 'runtime'? what a nice inconsistency,
-                // basically only serves to waste time in build logic!
-                configuration = project.configurations.findByName('testCompile')
-            }
-            assert configuration != null
-            if (project.plugins.hasPlugin(ShadowPlugin)) {
-                Configuration original = configuration
-                configuration = project.configurations.create('thirdPartyAudit')
-                configuration.extendsFrom(original, project.configurations.bundle)
-            }
-            if (compileOnly == null) {
-                classpath = configuration
-            } else {
-                classpath = project.files(configuration, compileOnly)
-            }
-
-            // we only want third party dependencies.
-            jars = configuration.fileCollection({ dependency ->
-                dependency.group.startsWith("org.elasticsearch") == false
-            });
-
-            // we don't want provided dependencies, which we have already scanned. e.g. don't
-            // scan ES core's dependencies for every single plugin
-            if (compileOnly != null) {
-                jars -= compileOnly
-            }
-            inputs.files(jars)
-            onlyIf { jars.isEmpty() == false }
-        }
-    }
-
-    /**
-     * classes that should be excluded from the scan,
-     * e.g. because we know what sheisty stuff those particular classes are up to.
-     */
-    public void setExcludes(String[] classes) {
-        for (String s : classes) {
-            if (s.indexOf('*') != -1) {
-                throw new IllegalArgumentException("illegal third party audit exclusion: '" + s + "', wildcards are not permitted!");
-            }
-        }
-        excludes = classes.sort();
-    }
-
-    /**
-     * Returns current list of exclusions.
-     */
-    @Input
-    public List<String> getExcludes() {
-        return excludes;
-    }
-
-    // yes, we parse Uwe Schindler's errors to find missing classes, and to keep a continuous audit. Just don't let him know!
-    static final Pattern MISSING_CLASS_PATTERN =
-        Pattern.compile(/WARNING: The referenced class '(.*)' cannot be loaded\. Please fix the classpath\!/);
-
-    static final Pattern VIOLATION_PATTERN =
-        Pattern.compile(/\s\sin ([a-zA-Z0-9\$\.]+) \(.*\)/);
-
-    // we log everything and capture errors and handle them with our whitelist
-    // this is important, as we detect stale whitelist entries, workaround forbidden apis bugs,
-    // and it also allows whitelisting missing classes!
-    static class EvilLogger extends DefaultLogger {
-        final Set<String> missingClasses = new TreeSet<>();
-        final Map<String,List<String>> violations = new TreeMap<>();
-        String previousLine = null;
-
-        @Override
-        public void messageLogged(BuildEvent event) {
-            if (event.getTask().getClass() == de.thetaphi.forbiddenapis.ant.AntTask.class) {
-                if (event.getPriority() == Project.MSG_WARN) {
-                    Matcher m = MISSING_CLASS_PATTERN.matcher(event.getMessage());
-                    if (m.matches()) {
-                        missingClasses.add(m.group(1).replace('.', '/') + ".class");
-                    }
-
-                    // Reset the priority of the event to DEBUG, so it doesn't
-                    // pollute the build output
-                    event.setMessage(event.getMessage(), Project.MSG_DEBUG);
-                } else if (event.getPriority() == Project.MSG_ERR) {
-                    Matcher m = VIOLATION_PATTERN.matcher(event.getMessage());
-                    if (m.matches()) {
-                        String violation = previousLine + '\n' + event.getMessage();
-                        String clazz = m.group(1).replace('.', '/') + ".class";
-                        List<String> current = violations.get(clazz);
-                        if (current == null) {
-                            current = new ArrayList<>();
-                            violations.put(clazz, current);
-                        }
-                        current.add(violation);
-                    }
-                    previousLine = event.getMessage();
-                }
-            }
-            super.messageLogged(event);
-        }
-    }
-
-    @Override
-    protected BuildLogger makeLogger(PrintStream stream, int outputLevel) {
-        DefaultLogger log = new EvilLogger();
-        log.errorPrintStream = stream;
-        log.outputPrintStream = stream;
-        log.messageOutputLevel = outputLevel;
-        return log;
-    }
-
-    @Override
-    protected void runAnt(AntBuilder ant) {
-        ant.project.addTaskDefinition('thirdPartyAudit', de.thetaphi.forbiddenapis.ant.AntTask);
-
-        // print which jars we are going to scan, always
-        // this is not the time to try to be succinct! Forbidden will print plenty on its own!
-        Set<String> names = new TreeSet<>();
-        for (File jar : jars) {
-            names.add(jar.getName());
-        }
-
-        // TODO: forbidden-apis + zipfileset gives O(n^2) behavior unless we dump to a tmpdir first,
-        // and then remove our temp dir afterwards. don't complain: try it yourself.
-        // we don't use gradle temp dir handling, just google it, or try it yourself.
-
-        File tmpDir = new File(project.buildDir, 'tmp/thirdPartyAudit');
-
-        // clean up any previous mess (if we failed), then unzip everything to one directory
-        ant.delete(dir: tmpDir.getAbsolutePath());
-        tmpDir.mkdirs();
-        for (File jar : jars) {
-            ant.unzip(src: jar.getAbsolutePath(), dest: tmpDir.getAbsolutePath());
-        }
-
-        // convert exclusion class names to binary file names
-        List<String> excludedFiles = excludes.collect {it.replace('.', '/') + ".class"}
-        Set<String> excludedSet = new TreeSet<>(excludedFiles);
-
-        // jarHellReprise
-        Set<String> sheistySet = getSheistyClasses(tmpDir.toPath());
-
-        try {
-            ant.thirdPartyAudit(failOnUnsupportedJava: false,
-                            failOnMissingClasses: false,
-                            classpath: classpath.asPath) {
-                fileset(dir: tmpDir)
-                signatures {
-                    string(value: getClass().getResourceAsStream('/forbidden/third-party-audit.txt').getText('UTF-8'))
-                }
-            }
-        } catch (BuildException ignore) {}
-
-        EvilLogger evilLogger = null;
-        for (BuildListener listener : ant.project.getBuildListeners()) {
-            if (listener instanceof EvilLogger) {
-                evilLogger = (EvilLogger) listener;
-                break;
-            }
-        }
-        assert evilLogger != null;
-
-        // keep our whitelist up to date
-        Set<String> bogusExclusions = new TreeSet<>(excludedSet);
-        bogusExclusions.removeAll(sheistySet);
-        bogusExclusions.removeAll(evilLogger.missingClasses);
-        bogusExclusions.removeAll(evilLogger.violations.keySet());
-        if (!bogusExclusions.isEmpty()) {
-            throw new IllegalStateException("Invalid exclusions, nothing is wrong with these classes: " + bogusExclusions);
-        }
-
-        // don't duplicate classes with the JDK
-        sheistySet.removeAll(excludedSet);
-        if (!sheistySet.isEmpty()) {
-            throw new IllegalStateException("JAR HELL WITH JDK! " + sheistySet);
-        }
-
-        // don't allow a broken classpath
-        evilLogger.missingClasses.removeAll(excludedSet);
-        if (!evilLogger.missingClasses.isEmpty()) {
-            throw new IllegalStateException("CLASSES ARE MISSING! " + evilLogger.missingClasses);
-        }
-
-        // don't use internal classes
-        evilLogger.violations.keySet().removeAll(excludedSet);
-        if (!evilLogger.violations.isEmpty()) {
-            throw new IllegalStateException("VIOLATIONS WERE FOUND! " + evilLogger.violations);
-        }
-
-        // clean up our mess (if we succeed)
-        ant.delete(dir: tmpDir.getAbsolutePath());
-
-        successMarker.setText("", 'UTF-8')
-    }
-
-    /**
-     * check for sheisty classes: if they also exist in the extensions classloader, its jar hell with the jdk!
-     */
-    private Set<String> getSheistyClasses(Path root) {
-        // system.parent = extensions loader.
-        // note: for jigsaw, this evilness will need modifications (e.g. use jrt filesystem!).
-        // but groovy/gradle needs to work at all first!
-        ClassLoader ext = ClassLoader.getSystemClassLoader().getParent();
-        assert ext != null;
-
-        Set<String> sheistySet = new TreeSet<>();
-        Files.walkFileTree(root, new SimpleFileVisitor<Path>() {
-            @Override
-            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
-                String entry = root.relativize(file).toString().replace('\\', '/');
-                if (entry.endsWith(".class")) {
-                    if (ext.getResource(entry) != null) {
-                        sheistySet.add(entry);
-                    }
-                }
-                return FileVisitResult.CONTINUE;
-            }
-        });
-        return sheistySet;
-    }
-}

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy

@@ -53,6 +53,8 @@ public class StandaloneRestTestPlugin implements Plugin<Project> {
 
         // only setup tests to build
         project.sourceSets.create('test')
+        // create a compileOnly configuration as others might expect it
+        project.configurations.create("compileOnly")
         project.dependencies.add('testCompile', "org.elasticsearch.test:framework:${VersionProperties.elasticsearch}")
 
         project.eclipse.classpath.sourceSets = [project.sourceSets.test]

buildSrc/src/main/java/org/elasticsearch/gradle/JdkJarHellCheck.java

@@ -0,0 +1,81 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle;
+
+import java.io.IOException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+public class JdkJarHellCheck {
+
+    private Set<String> detected = new HashSet<>();
+
+    private void scanForJDKJarHell(Path root) throws IOException {
+        // system.parent = extensions loader.
+        // note: for jigsaw, this evilness will need modifications (e.g. use jrt filesystem!)
+        ClassLoader ext = ClassLoader.getSystemClassLoader().getParent();
+        assert ext != null;
+
+        Files.walkFileTree(root, new SimpleFileVisitor<Path>() {
+            @Override
+            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
+                String entry = root.relativize(file).toString().replace('\\', '/');
+                if (entry.endsWith(".class")) {
+                    if (ext.getResource(entry) != null) {
+                        detected.add(
+                            entry
+                                .replace("/", ".")
+                                .replace(".class","")
+                        );
+                    }
+                }
+                return FileVisitResult.CONTINUE;
+            }
+        });
+    }
+
+    public Set<String> getDetected() {
+        return Collections.unmodifiableSet(detected);
+    }
+
+    public static void main(String[] argv) throws IOException {
+        JdkJarHellCheck checker = new JdkJarHellCheck();
+        for (String location : argv) {
+            Path path = Paths.get(location);
+            if (Files.exists(path) == false) {
+                throw new IllegalArgumentException("Path does not exist: " + path);
+            }
+            checker.scanForJDKJarHell(path);
+        }
+        if (checker.getDetected().isEmpty()) {
+            System.exit(0);
+        } else {
+            checker.getDetected().forEach(System.out::println);
+            System.exit(1);
+        }
+    }
+
+}

buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ForbiddenApisCliTask.java

@@ -18,10 +18,9 @@
  */
 package org.elasticsearch.gradle.precommit;
 
-import de.thetaphi.forbiddenapis.cli.CliMain;
-import org.gradle.api.Action;
 import org.gradle.api.DefaultTask;
 import org.gradle.api.JavaVersion;
+import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.logging.Logger;
 import org.gradle.api.logging.Logging;
@@ -29,6 +28,7 @@ import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.InputFiles;
 import org.gradle.api.tasks.OutputFile;
 import org.gradle.api.tasks.SkipWhenEmpty;
+import org.gradle.api.tasks.SourceSet;
 import org.gradle.api.tasks.TaskAction;
 import org.gradle.process.JavaExecSpec;
 
@@ -50,7 +50,8 @@ public class ForbiddenApisCliTask extends DefaultTask {
     private Set<String> suppressAnnotations = new LinkedHashSet<>();
     private JavaVersion targetCompatibility;
     private FileCollection classesDirs;
-    private Action<JavaExecSpec> execAction;
+    private SourceSet sourceSet;
+    private String javaHome;
 
     @Input
     public JavaVersion getTargetCompatibility() {
@@ -69,14 +70,6 @@ public class ForbiddenApisCliTask extends DefaultTask {
         }
     }
 
-    public Action<JavaExecSpec> getExecAction() {
-        return execAction;
-    }
-
-    public void setExecAction(Action<JavaExecSpec> execAction) {
-        this.execAction = execAction;
-    }
-
     @OutputFile
     public File getMarkerFile() {
         return new File(
@@ -131,11 +124,41 @@ public class ForbiddenApisCliTask extends DefaultTask {
         this.suppressAnnotations = suppressAnnotations;
     }
 
+    @InputFiles
+    public FileCollection getClassPathFromSourceSet() {
+        return getProject().files(
+            sourceSet.getCompileClasspath(),
+            sourceSet.getRuntimeClasspath()
+        );
+    }
+
+    public void setSourceSet(SourceSet sourceSet) {
+        this.sourceSet = sourceSet;
+    }
+
+    @InputFiles
+    public Configuration getForbiddenAPIsConfiguration() {
+        return getProject().getConfigurations().getByName("forbiddenApisCliJar");
+    }
+
+    @Input
+    public String getJavaHome() {
+        return javaHome;
+    }
+
+    public void setJavaHome(String javaHome) {
+        this.javaHome = javaHome;
+    }
+
     @TaskAction
     public void runForbiddenApisAndWriteMarker() throws IOException {
         getProject().javaexec((JavaExecSpec spec) -> {
-            execAction.execute(spec);
-            spec.setMain(CliMain.class.getName());
+            spec.classpath(
+                getForbiddenAPIsConfiguration(),
+                getClassPathFromSourceSet()
+            );
+            spec.setExecutable(getJavaHome() + "/bin/java");
+            spec.setMain("de.thetaphi.forbiddenapis.cli.CliMain");
             // build the command line
             getSignaturesFiles().forEach(file -> spec.args("-f", file.getAbsolutePath()));
             getSuppressAnnotations().forEach(annotation -> spec.args("--suppressannotation", annotation));

buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java

@@ -0,0 +1,288 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.precommit;
+
+import org.apache.commons.io.output.NullOutputStream;
+import org.elasticsearch.gradle.JdkJarHellCheck;
+import org.elasticsearch.test.NamingConventionsCheck;
+import org.gradle.api.DefaultTask;
+import org.gradle.api.GradleException;
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.file.FileCollection;
+import org.gradle.api.tasks.Input;
+import org.gradle.api.tasks.InputFile;
+import org.gradle.api.tasks.InputFiles;
+import org.gradle.api.tasks.OutputDirectory;
+import org.gradle.api.tasks.StopExecutionException;
+import org.gradle.api.tasks.TaskAction;
+import org.gradle.process.ExecResult;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+public class ThirdPartyAuditTask extends DefaultTask {
+
+    private static final Pattern MISSING_CLASS_PATTERN = Pattern.compile(
+        "WARNING: The referenced class '(.*)' cannot be loaded\\. Please fix the classpath!"
+    );
+
+    private static final Pattern VIOLATION_PATTERN = Pattern.compile(
+        "\\s\\sin ([a-zA-Z0-9$.]+) \\(.*\\)"
+    );
+
+    /**
+     * patterns for classes to exclude, because we understand their issues
+     */
+    private Set<String> excludes = new TreeSet<>();
+
+    private File signatureFile;
+
+    private String javaHome;
+
+    @InputFiles
+    public Configuration getForbiddenAPIsConfiguration() {
+        return getProject().getConfigurations().getByName("forbiddenApisCliJar");
+    }
+
+    @InputFile
+    public File getSignatureFile() {
+        return signatureFile;
+    }
+
+    public void setSignatureFile(File signatureFile) {
+        this.signatureFile = signatureFile;
+    }
+
+    @InputFiles
+    public Configuration getRuntimeConfiguration() {
+        Configuration runtime = getProject().getConfigurations().findByName("runtime");
+        if (runtime == null) {
+            return getProject().getConfigurations().getByName("testCompile");
+        }
+        return runtime;
+    }
+
+    @Input
+    public String getJavaHome() {
+        return javaHome;
+    }
+
+    public void setJavaHome(String javaHome) {
+        this.javaHome = javaHome;
+    }
+
+    @InputFiles
+    public Configuration getCompileOnlyConfiguration() {
+        return getProject().getConfigurations().getByName("compileOnly");
+    }
+
+    @OutputDirectory
+    public File getJarExpandDir() {
+        return new File(
+            new File(getProject().getBuildDir(), "precommit/thirdPartyAudit"),
+            getName()
+        );
+    }
+
+    public void setExcludes(String... classes) {
+        excludes.clear();
+        for (String each : classes) {
+            if (each.indexOf('*') != -1) {
+                throw new IllegalArgumentException("illegal third party audit exclusion: '" + each + "', wildcards are not permitted!");
+            }
+            excludes.add(each);
+        }
+    }
+
+    @Input
+    public Set<String> getExcludes() {
+        return Collections.unmodifiableSet(excludes);
+    }
+
+    @TaskAction
+    public void runThirdPartyAudit() throws IOException {
+        FileCollection jars = getJarsToScan();
+
+        extractJars(jars);
+
+        final String forbiddenApisOutput = runForbiddenAPIsCli();
+
+        final Set<String> missingClasses = new TreeSet<>();
+        Matcher missingMatcher = MISSING_CLASS_PATTERN.matcher(forbiddenApisOutput);
+        while (missingMatcher.find()) {
+            missingClasses.add(missingMatcher.group(1));
+        }
+
+        final Set<String> violationsClasses = new TreeSet<>();
+        Matcher violationMatcher = VIOLATION_PATTERN.matcher(forbiddenApisOutput);
+        while (violationMatcher.find()) {
+            violationsClasses.add(violationMatcher.group(1));
+        }
+
+        Set<String> jdkJarHellClasses = runJdkJarHellCheck();
+
+        assertNoPointlessExclusions(missingClasses, violationsClasses, jdkJarHellClasses);
+
+        assertNoMissingAndViolations(missingClasses, violationsClasses);
+
+        assertNoJarHell(jdkJarHellClasses);
+    }
+
+    private void extractJars(FileCollection jars) {
+        File jarExpandDir = getJarExpandDir();
+        jars.forEach(jar ->
+            getProject().copy(spec -> {
+                spec.from(getProject().zipTree(jar));
+                spec.into(jarExpandDir);
+            })
+        );
+    }
+
+    private void assertNoJarHell(Set<String> jdkJarHellClasses) {
+        jdkJarHellClasses.removeAll(excludes);
+        if (jdkJarHellClasses.isEmpty() == false) {
+            throw new IllegalStateException("Jar Hell with the JDK:" + formatClassList(jdkJarHellClasses));
+        }
+    }
+
+    private void assertNoMissingAndViolations(Set<String> missingClasses, Set<String> violationsClasses) {
+        missingClasses.removeAll(excludes);
+        violationsClasses.removeAll(excludes);
+        String missingText = formatClassList(missingClasses);
+        String violationsText = formatClassList(violationsClasses);
+        if (missingText.isEmpty() && violationsText.isEmpty()) {
+            getLogger().info("Third party audit passed successfully");
+        } else {
+            throw new IllegalStateException(
+                "Audit of third party dependencies failed:\n" +
+                    (missingText.isEmpty() ?  "" : "Missing classes:\n" + missingText) +
+                    (violationsText.isEmpty() ? "" : "Classes with violations:\n" + violationsText)
+            );
+        }
+    }
+
+    private void assertNoPointlessExclusions(Set<String> missingClasses, Set<String> violationsClasses, Set<String> jdkJarHellClasses) {
+        // keep our whitelist up to date
+        Set<String> bogusExclusions = new TreeSet<>(excludes);
+        bogusExclusions.removeAll(missingClasses);
+        bogusExclusions.removeAll(jdkJarHellClasses);
+        bogusExclusions.removeAll(violationsClasses);
+        if (bogusExclusions.isEmpty() == false) {
+            throw new IllegalStateException(
+                "Invalid exclusions, nothing is wrong with these classes: " + formatClassList(bogusExclusions)
+            );
+        }
+    }
+
+    private String runForbiddenAPIsCli() throws IOException {
+        ByteArrayOutputStream errorOut = new ByteArrayOutputStream();
+        getProject().javaexec(spec -> {
+            spec.setExecutable(javaHome + "/bin/java");
+            spec.classpath(
+                getForbiddenAPIsConfiguration(),
+                getRuntimeConfiguration(),
+                getCompileOnlyConfiguration()
+            );
+            spec.setMain("de.thetaphi.forbiddenapis.cli.CliMain");
+            spec.args(
+                "-f", getSignatureFile().getAbsolutePath(),
+                "-d", getJarExpandDir(),
+                "--allowmissingclasses"
+            );
+            spec.setErrorOutput(errorOut);
+            if (getLogger().isInfoEnabled() == false) {
+                spec.setStandardOutput(new NullOutputStream());
+            }
+            spec.setIgnoreExitValue(true);
+        });
+        final String forbiddenApisOutput;
+        try (ByteArrayOutputStream outputStream = errorOut) {
+            forbiddenApisOutput = outputStream.toString(StandardCharsets.UTF_8.name());
+        }
+        if (getLogger().isInfoEnabled()) {
+            getLogger().info(forbiddenApisOutput);
+        }
+        return forbiddenApisOutput;
+    }
+
+    private FileCollection getJarsToScan() {
+        FileCollection jars = getRuntimeConfiguration()
+            .fileCollection(dep -> dep.getGroup().startsWith("org.elasticsearch") == false);
+        Configuration compileOnlyConfiguration = getCompileOnlyConfiguration();
+        // don't scan provided dependencies that we already scanned, e.x. don't scan cores dependencies for every plugin
+        if (compileOnlyConfiguration != null) {
+            jars.minus(compileOnlyConfiguration);
+        }
+        if (jars.isEmpty()) {
+            throw new StopExecutionException("No jars to scan");
+        }
+        return jars;
+    }
+
+    private String formatClassList(Set<String> classList) {
+        return classList.stream()
+            .map(name -> "  * " + name)
+            .collect(Collectors.joining("\n"));
+    }
+
+    private Set<String> runJdkJarHellCheck() throws IOException {
+        ByteArrayOutputStream standardOut = new ByteArrayOutputStream();
+        ExecResult execResult = getProject().javaexec(spec -> {
+            URL location = NamingConventionsCheck.class.getProtectionDomain().getCodeSource().getLocation();
+            if (location.getProtocol().equals("file") == false) {
+                throw new GradleException("Unexpected location for NamingConventionCheck class: " + location);
+            }
+            try {
+                spec.classpath(
+                    location.toURI().getPath(),
+                    getRuntimeConfiguration(),
+                    getCompileOnlyConfiguration()
+                );
+            } catch (URISyntaxException e) {
+                throw new AssertionError(e);
+            }
+            spec.setMain(JdkJarHellCheck.class.getName());
+            spec.args(getJarExpandDir());
+            spec.setIgnoreExitValue(true);
+            spec.setExecutable(javaHome + "/bin/java");
+            spec.setStandardOutput(standardOut);
+        });
+        if (execResult.getExitValue() == 0) {
+            return Collections.emptySet();
+        }
+        final String jdkJarHellCheckList;
+        try (ByteArrayOutputStream outputStream = standardOut) {
+            jdkJarHellCheckList = outputStream.toString(StandardCharsets.UTF_8.name());
+        }
+        return new TreeSet<>(Arrays.asList(jdkJarHellCheckList.split("\\r?\\n")));
+    }
+
+
+}

plugins/discovery-azure-classic/build.gradle

@@ -128,7 +128,7 @@ thirdPartyAudit.excludes = [
 ]
 
 // jarhell with jdk (intentionally, because jaxb was removed from default modules in java 9)
-if (JavaVersion.current() <= JavaVersion.VERSION_1_8) {
+if (project.runtimeJavaVersion <= JavaVersion.VERSION_1_8) {
   thirdPartyAudit.excludes += [
     'javax.xml.bind.Binder',
     'javax.xml.bind.ContextFinder$1',

plugins/discovery-ec2/build.gradle

@@ -87,7 +87,7 @@ thirdPartyAudit.excludes = [
   'org.apache.log.Logger',
 ]
 
-if (JavaVersion.current() > JavaVersion.VERSION_1_8) {
+if (project.runtimeJavaVersion > JavaVersion.VERSION_1_8) {
   thirdPartyAudit.excludes += [
     'javax.xml.bind.DatatypeConverter',
     'javax.xml.bind.JAXBContext'

plugins/ingest-attachment/build.gradle

@@ -2106,7 +2106,27 @@ thirdPartyAudit.excludes = [
   'ucar.nc2.dataset.NetcdfDataset'
 ]
 
-if (JavaVersion.current() > JavaVersion.VERSION_1_8) {
+if (project.runtimeJavaVersion == JavaVersion.VERSION_1_8) {
+    thirdPartyAudit.excludes += [
+            // TODO: Why is this needed ?
+            'com.sun.javadoc.ClassDoc',
+            'com.sun.javadoc.Doc',
+            'com.sun.javadoc.Doclet',
+            'com.sun.javadoc.ExecutableMemberDoc',
+            'com.sun.javadoc.FieldDoc',
+            'com.sun.javadoc.MethodDoc',
+            'com.sun.javadoc.PackageDoc',
+            'com.sun.javadoc.Parameter',
+            'com.sun.javadoc.ProgramElementDoc',
+            'com.sun.javadoc.RootDoc',
+            'com.sun.javadoc.SourcePosition',
+            'com.sun.javadoc.Tag',
+            'com.sun.javadoc.Type',
+            'com.sun.tools.javadoc.Main'
+    ]
+}
+
+if (project.runtimeJavaVersion > JavaVersion.VERSION_1_8) {
   thirdPartyAudit.excludes += [
     'javax.activation.ActivationDataFlavor',
     'javax.activation.CommandMap',

plugins/repository-hdfs/build.gradle

@@ -582,6 +582,25 @@ thirdPartyAudit.excludes = [
   'com.squareup.okhttp.ResponseBody'
 ]
 
-if (JavaVersion.current() > JavaVersion.VERSION_1_8) {
+if (project.runtimeJavaVersion > JavaVersion.VERSION_1_8) {
   thirdPartyAudit.excludes += ['javax.xml.bind.annotation.adapters.HexBinaryAdapter']
 }
+
+if (project.runtimeJavaVersion == JavaVersion.VERSION_1_8) {
+  thirdPartyAudit.excludes += [
+          // TODO: Why is this needed ?
+          'com.sun.javadoc.AnnotationDesc',
+          'com.sun.javadoc.AnnotationTypeDoc',
+          'com.sun.javadoc.ClassDoc',
+          'com.sun.javadoc.ConstructorDoc',
+          'com.sun.javadoc.Doc',
+          'com.sun.javadoc.DocErrorReporter',
+          'com.sun.javadoc.FieldDoc',
+          'com.sun.javadoc.LanguageVersion',
+          'com.sun.javadoc.MethodDoc',
+          'com.sun.javadoc.PackageDoc',
+          'com.sun.javadoc.ProgramElementDoc',
+          'com.sun.javadoc.RootDoc',
+          'com.sun.tools.doclets.standard.Standard'
+  ]
+}

plugins/repository-s3/build.gradle

@@ -447,7 +447,7 @@ thirdPartyAudit.excludes = [
 ]
 
 // jarhell with jdk (intentionally, because jaxb was removed from default modules in java 9)
-if (JavaVersion.current() <= JavaVersion.VERSION_1_8) {
+if (project.runtimeJavaVersion <= JavaVersion.VERSION_1_8) {
   thirdPartyAudit.excludes += [
     'javax.xml.bind.Binder',
     'javax.xml.bind.ContextFinder$1',

server/build.gradle

@@ -304,17 +304,22 @@ thirdPartyAudit.excludes = [
   'com.google.common.geometry.S2LatLng',
 ]
 
-if (JavaVersion.current() <= JavaVersion.VERSION_1_8) {
-  // Used by Log4J 2.11.1
+if (project.runtimeJavaVersion <= JavaVersion.VERSION_1_8) {
   thirdPartyAudit.excludes += [
-    'java.io.ObjectInputFilter',
-    'java.io.ObjectInputFilter$Config',
-    'java.io.ObjectInputFilter$FilterInfo',
-    'java.io.ObjectInputFilter$Status'
+          // Used by Log4J 2.11.1
+            'java.io.ObjectInputFilter',
+            'java.io.ObjectInputFilter$Config',
+            'java.io.ObjectInputFilter$FilterInfo',
+            'java.io.ObjectInputFilter$Status',
+          // added in 9
+            'java.lang.ProcessHandle',
+            'java.lang.StackWalker',
+            'java.lang.StackWalker$Option',
+            'java.lang.StackWalker$StackFrame'
   ]
 }
 
-if (JavaVersion.current() > JavaVersion.VERSION_1_8) {
+if (project.runtimeJavaVersion > JavaVersion.VERSION_1_8) {
   thirdPartyAudit.excludes += ['javax.xml.bind.DatatypeConverter']
 }
 

test/logger-usage/build.gradle

@@ -44,7 +44,7 @@ thirdPartyAudit.excludes = [
   'org.osgi.framework.wiring.BundleWiring'
 ]
 
-if (JavaVersion.current() <= JavaVersion.VERSION_1_8) {
+if (project.runtimeJavaVersion <= JavaVersion.VERSION_1_8) {
   // Used by Log4J 2.11.1
   thirdPartyAudit.excludes += [
     'java.io.ObjectInputFilter',
@@ -52,4 +52,13 @@ if (JavaVersion.current() <= JavaVersion.VERSION_1_8) {
     'java.io.ObjectInputFilter$FilterInfo',
     'java.io.ObjectInputFilter$Status'
   ]
+}
+
+if (project.runtimeJavaVersion == JavaVersion.VERSION_1_8) {
+  thirdPartyAudit.excludes += [
+     'java.lang.ProcessHandle',
+     'java.lang.StackWalker',
+     'java.lang.StackWalker$Option',
+     'java.lang.StackWalker$StackFrame'
+  ]
 }

x-pack/plugin/security/build.gradle

@@ -242,7 +242,7 @@ thirdPartyAudit.excludes = [
         'javax.persistence.EntityManagerFactory',
         'javax.persistence.EntityTransaction',
         'javax.persistence.LockModeType',
-        'javax/persistence/Query',
+        'javax.persistence.Query',
         // [missing classes] OpenSAML storage and HttpClient cache have optional memcache support
         'net.spy.memcached.CASResponse',
         'net.spy.memcached.CASValue',
@@ -266,7 +266,7 @@ thirdPartyAudit.excludes = [
         'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1',
 ]
 
-if (JavaVersion.current() > JavaVersion.VERSION_1_8) {
+if (project.runtimeJavaVersion > JavaVersion.VERSION_1_8) {
     thirdPartyAudit.excludes += [
         'javax.xml.bind.JAXBContext',
         'javax.xml.bind.JAXBElement',

x-pack/plugin/sql/sql-action/build.gradle

@@ -140,7 +140,7 @@ thirdPartyAudit.excludes = [
         'org.zeromq.ZMQ'
 ]
 
-if (JavaVersion.current() <= JavaVersion.VERSION_1_8) {
+if (project.runtimeJavaVersion <= JavaVersion.VERSION_1_8) {
     // Used by Log4J 2.11.1
     thirdPartyAudit.excludes += [
       'java.io.ObjectInputFilter',
@@ -148,4 +148,13 @@ if (JavaVersion.current() <= JavaVersion.VERSION_1_8) {
       'java.io.ObjectInputFilter$FilterInfo',
       'java.io.ObjectInputFilter$Status'
     ]
+}
+
+if (project.runtimeJavaVersion == JavaVersion.VERSION_1_8) {
+    thirdPartyAudit.excludes += [
+      'java.lang.ProcessHandle',
+      'java.lang.StackWalker',
+      'java.lang.StackWalker$Option',
+      'java.lang.StackWalker$StackFrame'
+    ]
 }

x-pack/plugin/watcher/build.gradle

@@ -68,7 +68,7 @@ thirdPartyAudit.excludes = [
 ]
 
 // pulled in as external dependency to work on java 9
-if (JavaVersion.current() <= JavaVersion.VERSION_1_8) {
+if (project.runtimeJavaVersion <= JavaVersion.VERSION_1_8) {
     thirdPartyAudit.excludes += [
         'com.sun.activation.registries.MailcapParseException',
         'javax.activation.ActivationDataFlavor',