Build: Add AntTask to simplify controlling logging when running ant from gradle

This new task allows setting code, similar to a doLast or doFirst,
except it is specifically geared at running ant (and thus called doAnt).
It adjusts the ant logging while running the ant so that the log
level/behavior can be tweaked, and automatically buffers based on gradle
logging level, and dumps the ant output upon failure.

build.gradle

@@ -123,17 +123,6 @@ subprojects {
       }
     }
   }
-  // For reasons we don't fully understand yet, external dependencies are not picked up by Ant's optional tasks.
-  // But you can easily do it in another way.
-  // Only if your buildscript and Ant's optional task need the same library would you have to define it twice.
-  // https://docs.gradle.org/current/userguide/organizing_build_logic.html
-  configurations {
-    buildTools
-  }
-  dependencies {
-    buildTools 'de.thetaphi:forbiddenapis:2.0'
-    buildTools 'org.apache.rat:apache-rat:0.11'
-  }
 }
 
 // Ensure similar tasks in dependent projects run first. The projectsEvaluated here is

buildSrc/build.gradle

@@ -63,6 +63,7 @@ dependencies {
   compile 'com.perforce:p4java:2012.3.551082' // THIS IS SUPPOSED TO BE OPTIONAL IN THE FUTURE....
   compile 'de.thetaphi:forbiddenapis:2.0'
   compile 'com.bmuschko:gradle-nexus-plugin:2.3.1'
+  compile 'org.apache.rat:apache-rat:0.11'
 }
 
 processResources {

buildSrc/src/main/groovy/org/elasticsearch/gradle/AntTask.groovy

@@ -0,0 +1,111 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle
+
+import org.apache.tools.ant.BuildException
+import org.apache.tools.ant.BuildListener
+import org.apache.tools.ant.BuildLogger
+import org.apache.tools.ant.DefaultLogger
+import org.apache.tools.ant.Project
+import org.gradle.api.DefaultTask
+import org.gradle.api.GradleException
+import org.gradle.api.tasks.Input
+import org.gradle.api.tasks.Optional
+import org.gradle.api.tasks.TaskAction
+
+import java.nio.charset.Charset
+
+/**
+ * A task which will run ant commands.
+ *
+ * Logging for the task is customizable for subclasses by overriding makeLogger.
+ */
+public class AntTask extends DefaultTask {
+
+    /**
+     * A buffer that will contain the output of the ant code run,
+     * if the output was not already written directly to stdout.
+     */
+    public final ByteArrayOutputStream outputBuffer = new ByteArrayOutputStream()
+
+    @TaskAction
+    final void executeTask() {
+        // capture the current loggers
+        List<BuildLogger> savedLoggers = new ArrayList<>();
+        for (BuildListener l : project.ant.project.getBuildListeners()) {
+            if (l instanceof BuildLogger) {
+                savedLoggers.add(l);
+            }
+        }
+        // remove them
+        for (BuildLogger l : savedLoggers) {
+            project.ant.project.removeBuildListener(l)
+        }
+
+        final int outputLevel = logger.isDebugEnabled() ? Project.MSG_DEBUG : Project.MSG_INFO
+        final PrintStream stream = useStdout() ? System.out : new PrintStream(outputBuffer, true, Charset.defaultCharset().name())
+        BuildLogger antLogger = makeLogger(stream, outputLevel)
+
+        // now run the command with just our logger
+        project.ant.project.addBuildListener(antLogger)
+        try {
+            runAnt(project.ant)
+        } catch (BuildException e) {
+            // ant failed, so see if we have buffered output to emit, then rethrow the failure
+            String buffer = outputBuffer.toString()
+            if (buffer.isEmpty() == false) {
+                logger.error("=== Ant output ===\n${buffer}")
+            }
+            throw e
+        } finally {
+            project.ant.project.removeBuildListener(antLogger)
+            // add back the old loggers before returning
+            for (BuildLogger l : savedLoggers) {
+                project.ant.project.addBuildListener(l)
+            }
+        }
+    }
+
+    /** Runs the doAnt closure. This can be overridden by subclasses instead of having to set a closure. */
+    protected void runAnt(AntBuilder ant) {
+        if (doAnt == null) {
+            throw new GradleException("Missing doAnt for ${name}")
+        }
+        doAnt(ant)
+    }
+
+    /** Create the logger the ant runner will use, with the given stream for error/output. */
+    protected BuildLogger makeLogger(PrintStream stream, int outputLevel) {
+        return new DefaultLogger(
+            errorPrintStream: stream,
+            outputPrintStream: stream,
+            messageOutputLevel: outputLevel)
+    }
+
+    /**
+     * Returns true if the ant logger should write to stdout, or false if to the buffer.
+     * The default implementation writes to the buffer when gradle info logging is disabled.
+     */
+    protected boolean useStdout() {
+        return logger.isInfoEnabled()
+    }
+
+
+}

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/LicenseHeadersTask.groovy

@@ -18,34 +18,33 @@
  */
 package org.elasticsearch.gradle.precommit
 
-import java.nio.file.Files
-
-import org.gradle.api.DefaultTask
+import org.apache.rat.anttasks.Report
+import org.apache.rat.anttasks.SubstringLicenseMatcher
+import org.apache.rat.license.SimpleLicenseFamily
+import org.elasticsearch.gradle.AntTask
 import org.gradle.api.tasks.SourceSet
-import org.gradle.api.tasks.TaskAction
 
-import groovy.xml.NamespaceBuilder
-import groovy.xml.NamespaceBuilderSupport
+import java.nio.file.Files
 
 /**
  * Checks files for license headers.
  * <p>
  * This is a port of the apache lucene check
  */
-public class LicenseHeadersTask extends DefaultTask {
+public class LicenseHeadersTask extends AntTask {
 
     LicenseHeadersTask() {
         description = "Checks sources for missing, incorrect, or unacceptable license headers"
+
+        if (ant.project.taskDefinitions.contains('ratReport') == false) {
+            ant.project.addTaskDefinition('ratReport', Report)
+            ant.project.addDataTypeDefinition('substringMatcher', SubstringLicenseMatcher)
+            ant.project.addDataTypeDefinition('approvedLicense', SimpleLicenseFamily)
+        }
     }
 
-    @TaskAction
-    public void check() {
-        // load rat tasks
-        AntBuilder ant = new AntBuilder()
-        ant.typedef(resource:  "org/apache/rat/anttasks/antlib.xml",
-                    uri:       "antlib:org.apache.rat.anttasks",
-                    classpath: project.configurations.buildTools.asPath)
-        NamespaceBuilderSupport rat = NamespaceBuilder.newInstance(ant, "antlib:org.apache.rat.anttasks")
+    @Override
+    protected void runAnt(AntBuilder ant) {
 
         // create a file for the log to go to under reports/
         File reportDir = new File(project.buildDir, "reports/licenseHeaders")
@@ -54,7 +53,7 @@ public class LicenseHeadersTask extends DefaultTask {
         Files.deleteIfExists(reportFile.toPath())
                      
         // run rat, going to the file
-        rat.report(reportFile: reportFile.absolutePath, addDefaultLicenseMatchers: true) {
+        ant.ratReport(reportFile: reportFile.absolutePath, addDefaultLicenseMatchers: true) {
                // checks all the java sources (allJava)
                for (SourceSet set : project.sourceSets) {
                    for (File dir : set.allJava.srcDirs) {

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.groovy

@@ -18,6 +18,10 @@
  */
 package org.elasticsearch.gradle.precommit
 
+import org.apache.tools.ant.DefaultLogger
+import org.elasticsearch.gradle.AntTask
+import org.gradle.api.artifacts.Configuration
+
 import java.nio.file.Files
 import java.nio.file.FileVisitResult
 import java.nio.file.Path
@@ -35,7 +39,7 @@ import org.apache.tools.ant.Project
 /**
  * Basic static checking to keep tabs on third party JARs
  */
-public class ThirdPartyAuditTask extends DefaultTask {
+public class ThirdPartyAuditTask extends AntTask {
 
     // true to be lenient about MISSING CLASSES
     private boolean missingClasses;
@@ -46,6 +50,10 @@ public class ThirdPartyAuditTask extends DefaultTask {
     ThirdPartyAuditTask() {
         dependsOn(project.configurations.testCompile)
         description = "Checks third party JAR bytecode for missing classes, use of internal APIs, and other horrors'"
+
+        if (ant.project.taskDefinitions.contains('thirdPartyAudit') == false) {
+            ant.project.addTaskDefinition('thirdPartyAudit', de.thetaphi.forbiddenapis.ant.AntTask)
+        }
     }
 
     /** 
@@ -84,38 +92,35 @@ public class ThirdPartyAuditTask extends DefaultTask {
         return excludes;
     }
 
-    @TaskAction
-    public void check() {
-        AntBuilder ant = new AntBuilder()
+    @Override
+    protected BuildLogger makeLogger(PrintStream stream, int outputLevel) {
+        return new DefaultLogger(
+            errorPrintStream: stream,
+            outputPrintStream: stream,
+            // ignore passed in outputLevel for now, until we are filtering warning messages
+            messageOutputLevel: Project.MSG_ERR)
+    }
 
-        // we are noisy for many reasons, working around performance problems with forbidden-apis, dealing
-        // with warnings about missing classes, etc. so we use our own "quiet" AntBuilder
-        ant.project.buildListeners.each { listener ->
-            if (listener instanceof BuildLogger) {
-              listener.messageOutputLevel = Project.MSG_ERR;
-            }
-        };
-        
+    @Override
+    protected void runAnt(AntBuilder ant) {
         // we only want third party dependencies.
-        FileCollection jars = project.configurations.testCompile.fileCollection({ dependency -> 
+        FileCollection jars = project.configurations.testCompile.fileCollection({ dependency ->
             dependency.group.startsWith("org.elasticsearch") == false
         })
-        
+
         // we don't want provided dependencies, which we have already scanned. e.g. don't
         // scan ES core's dependencies for every single plugin
-        try {
-            jars -= project.configurations.getByName("provided")
-        } catch (UnknownConfigurationException ignored) {}
-        
+        Configuration provided = project.configurations.findByName('provided')
+        if (provided != null) {
+            jars -= provided
+        }
+
         // no dependencies matched, we are done
         if (jars.isEmpty()) {
             return;
         }
-        
-        ant.taskdef(name:      "thirdPartyAudit",
-                    classname: "de.thetaphi.forbiddenapis.ant.AntTask",
-                    classpath: project.configurations.buildTools.asPath)
-        
+
+
         // print which jars we are going to scan, always
         // this is not the time to try to be succinct! Forbidden will print plenty on its own!
         Set<String> names = new HashSet<>()
@@ -123,26 +128,26 @@ public class ThirdPartyAuditTask extends DefaultTask {
             names.add(jar.getName())
         }
         logger.error("[thirdPartyAudit] Scanning: " + names)
-        
+
         // warn that classes are missing
         // TODO: move these to excludes list!
         if (missingClasses) {
             logger.warn("[thirdPartyAudit] WARNING: CLASSES ARE MISSING! Expect NoClassDefFoundError in bug reports from users!")
         }
-        
-        // TODO: forbidden-apis + zipfileset gives O(n^2) behavior unless we dump to a tmpdir first, 
+
+        // TODO: forbidden-apis + zipfileset gives O(n^2) behavior unless we dump to a tmpdir first,
         // and then remove our temp dir afterwards. don't complain: try it yourself.
         // we don't use gradle temp dir handling, just google it, or try it yourself.
-        
+
         File tmpDir = new File(project.buildDir, 'tmp/thirdPartyAudit')
-        
+
         // clean up any previous mess (if we failed), then unzip everything to one directory
         ant.delete(dir: tmpDir.getAbsolutePath())
         tmpDir.mkdirs()
         for (File jar : jars) {
             ant.unzip(src: jar.getAbsolutePath(), dest: tmpDir.getAbsolutePath())
         }
-        
+
         // convert exclusion class names to binary file names
         String[] excludedFiles = new String[excludes.length];
         for (int i = 0; i < excludes.length; i++) {
@@ -152,12 +157,12 @@ public class ThirdPartyAuditTask extends DefaultTask {
                 throw new IllegalStateException("bogus thirdPartyAudit exclusion: '" + excludes[i] + "', not found in any dependency")
             }
         }
-        
+
         // jarHellReprise
         checkSheistyClasses(tmpDir.toPath(), new HashSet<>(Arrays.asList(excludedFiles)));
-        
-        ant.thirdPartyAudit(internalRuntimeForbidden: true, 
-                            failOnUnsupportedJava: false, 
+
+        ant.thirdPartyAudit(internalRuntimeForbidden: true,
+                            failOnUnsupportedJava: false,
                             failOnMissingClasses: !missingClasses,
                             classpath: project.configurations.testCompile.asPath) {
             fileset(dir: tmpDir, excludes: excludedFiles.join(','))
@@ -169,7 +174,7 @@ public class ThirdPartyAuditTask extends DefaultTask {
     /**
      * check for sheisty classes: if they also exist in the extensions classloader, its jar hell with the jdk!
      */
-    private void checkSheistyClasses(Path root, Set<String> excluded) {
+    protected void checkSheistyClasses(Path root, Set<String> excluded) {
         // system.parent = extensions loader.
         // note: for jigsaw, this evilness will need modifications (e.g. use jrt filesystem!). 
         // but groovy/gradle needs to work at all first!