Build: Add notice file generation (#23170)

This change improves the notice file present in our distributions to
include notice and license files from each included dependency.

closes #22546

NOTICE.txt

@@ -1,5 +1,5 @@
 Elasticsearch
-Copyright 2009-2016 Elasticsearch
+Copyright 2009-2017 Elasticsearch
 
 This product includes software developed by The Apache Software
 Foundation (http://www.apache.org/).

buildSrc/src/main/groovy/org/elasticsearch/gradle/NoticeTask.groovy

@@ -0,0 +1,78 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle
+
+import org.gradle.api.DefaultTask
+import org.gradle.api.Project
+import org.gradle.api.artifacts.Configuration
+import org.gradle.api.tasks.OutputFile
+import org.gradle.api.tasks.TaskAction
+
+/**
+ * A task to create a notice file which includes dependencies' notices.
+ */
+public class NoticeTask extends DefaultTask {
+
+    @OutputFile
+    File noticeFile = new File(project.buildDir, "notices/${name}/NOTICE.txt")
+
+    /** Configurations to inspect dependencies*/
+    private List<Project> dependencies = new ArrayList<>()
+
+    public NoticeTask() {
+        description = 'Create a notice file from dependencies'
+    }
+
+    /** Add notices from licenses found in the given project. */
+    public void dependencies(Project project) {
+        dependencies.add(project)
+    }
+
+    @TaskAction
+    public void generateNotice() {
+        StringBuilder output = new StringBuilder()
+        output.append(project.rootProject.file('NOTICE.txt').getText('UTF-8'))
+        output.append('\n\n')
+        Set<String> seen = new HashSet<>()
+        for (Project dep : dependencies) {
+            File licensesDir = new File(dep.projectDir, 'licenses')
+            if (licensesDir.exists() == false) continue
+            licensesDir.eachFileMatch({ it ==~ /.*-NOTICE\.txt/ && seen.contains(it) == false}) { File file ->
+                String name = file.name.substring(0, file.name.length() - '-NOTICE.txt'.length())
+                appendFile(file, name, 'NOTICE', output)
+                appendFile(new File(file.parentFile, "${name}-LICENSE.txt"), name, 'LICENSE', output)
+                seen.add(file.name)
+            }
+        }
+        noticeFile.setText(output.toString(), 'UTF-8')
+    }
+
+    static void appendFile(File file, String name, String type, StringBuilder output) {
+        String text = file.getText('UTF-8')
+        if (text.trim().isEmpty()) {
+            return
+        }
+        output.append('================================================================================\n')
+        output.append("${name} ${type}\n")
+        output.append('================================================================================\n')
+        output.append(text)
+        output.append('\n\n')
+    }
+}

distribution/build.gradle

@@ -23,6 +23,7 @@ import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.BuildPlugin
 import org.elasticsearch.gradle.EmptyDirTask
 import org.elasticsearch.gradle.MavenFilteringHack
+import org.elasticsearch.gradle.NoticeTask
 import org.elasticsearch.gradle.precommit.DependencyLicensesTask
 import org.elasticsearch.gradle.precommit.UpdateShasTask
 import org.elasticsearch.gradle.test.RunTask
@@ -39,6 +40,20 @@ buildscript {
   }
 }
 
+/*****************************************************************************
+ *                                Notice file                                *
+ *****************************************************************************/
+
+// integ test zip only uses core, so a different notice file is needed there
+task buildCoreNotice(type: NoticeTask) {
+  dependencies project(':core')
+}
+
+// other distributions include notices from modules as well, which are added below later
+task buildFullNotice(type: NoticeTask) {
+  dependencies project(':core')
+}
+
 /*****************************************************************************
  *                                  Modules                                  *
  *****************************************************************************/
@@ -54,6 +69,9 @@ ext.restTestExpansions = [
 // depend on it, but we don't actually configure it until here so we can do a single
 // loop over modules to also setup cross task dependencies and increment our modules counter
 project.rootProject.subprojects.findAll { it.path.startsWith(':modules:') }.each { Project module ->
+  buildFullNotice {
+    dependencies module
+  }
   buildModules {
     dependsOn({ project(module.path).bundlePlugin })
     into(module.name) {
@@ -172,10 +190,17 @@ subprojects {
 
     commonFiles = copySpec {
       from rootProject.projectDir
-      include 'NOTICE.txt'
       include 'LICENSE.txt'
       include 'README.textile'
     }
+
+    noticeFile = copySpec {
+      if (project.name == 'integ-test-zip') {
+        from buildCoreNotice
+      } else {
+        from buildFullNotice
+      }
+    }
   }
 
    /*****************************************************************************
@@ -224,6 +249,7 @@ configure(subprojects.findAll { ['zip', 'tar', 'integ-test-zip'].contains(it.nam
         }
       }
       with commonFiles
+      with noticeFile
       from('../src/main/resources') {
         include 'bin/*.exe'
       }
@@ -375,6 +401,7 @@ configure(subprojects.findAll { ['deb', 'rpm'].contains(it.name) }) {
         exclude 'LICENSE.txt'
       }
     }
+    with noticeFile
 
     configurationFile '/etc/elasticsearch/elasticsearch.yml'
     configurationFile '/etc/elasticsearch/jvm.options'