Add build time checks for package licenses

This commit adds some build time checks that the archive distributions
and package distributions contain the appropriate license and notice
files, and the package distributions contain the appropriate license
metadata.

distribution/archives/build.gradle

@@ -23,8 +23,12 @@ import org.elasticsearch.gradle.BuildPlugin
 import org.elasticsearch.gradle.EmptyDirTask
 import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.MavenFilteringHack
+import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.plugin.PluginBuildPlugin
 
+import java.nio.file.Files
+import java.nio.file.Path
+
 // need this so Zip/Tar tasks get basic defaults...
 apply plugin: 'base'
 
@@ -129,6 +133,9 @@ task buildOssTar(type: Tar) {
   with archiveFiles(modulesFiles(true), 'tar', true)
 }
 
+Closure tarExists = { it -> new File('/bin/tar').exists() || new File('/usr/bin/tar').exists() || new File('/usr/local/bin/tar').exists() }
+Closure unzipExists = { it -> new File('/bin/unzip').exists() || new File('/usr/bin/unzip').exists() || new File('/usr/local/bin/unzip').exists() }
+
 // This configures the default artifact for the distribution specific
 // subprojects. We have subprojects for two reasons:
 // 1. Gradle project substitutions can only bind to the default
@@ -144,27 +151,74 @@ subprojects {
     'default' buildDist
   }
 
-  // sanity checks if a archives can be extracted
-  File extractionDir = new File(buildDir, 'extracted')
-  task testExtraction(type: LoggedExec) {
+  // sanity checks if archives can be extracted
+  final File archiveExtractionDir
+  if (project.name.contains('tar')) {
+    archiveExtractionDir = new File(buildDir, 'tar-extracted')
+  } else {
+    assert project.name.contains('zip')
+    archiveExtractionDir = new File(buildDir, 'zip-extracted')
+  }
+  task checkExtraction(type: LoggedExec) {
     dependsOn buildDist
     doFirst {
-      project.delete(extractionDir)
-      extractionDir.mkdirs()
+      project.delete(archiveExtractionDir)
+      archiveExtractionDir.mkdirs()
     }
   }
-  if (project.name.contains('zip')) {
-    testExtraction {
-      onlyIf { new File('/bin/unzip').exists() || new File('/usr/bin/unzip').exists() || new File('/usr/local/bin/unzip').exists() }
-      commandLine 'unzip', "${-> buildDist.outputs.files.singleFile}", '-d', extractionDir
+  check.dependsOn checkExtraction
+  if (project.name.contains('tar')) {
+    checkExtraction {
+      onlyIf tarExists
+      commandLine 'tar', '-xvzf', "${-> buildDist.outputs.files.singleFile}", '-C', archiveExtractionDir
+    }
+  } else {
+    assert project.name.contains('zip')
+    checkExtraction {
+      onlyIf unzipExists
+      commandLine 'unzip', "${-> buildDist.outputs.files.singleFile}", '-d', archiveExtractionDir
+    }
+  }
+
+  final Closure toolExists
+  if (project.name.contains('tar')) {
+    toolExists = tarExists
+  } else {
+    assert project.name.contains('zip')
+    toolExists = unzipExists
+  }
+
+
+  task checkLicense {
+    dependsOn buildDist, checkExtraction
+    onlyIf toolExists
+    doLast {
+      final String licenseFilename
+      if (project.name.contains('oss-')) {
+        licenseFilename = "APACHE-LICENSE-2.0.txt"
+      } else {
+        licenseFilename = "ELASTIC-LICENSE.txt"
+      }
+      final List<String> licenseLines = Files.readAllLines(rootDir.toPath().resolve("licenses/" + licenseFilename))
+      final Path licensePath = archiveExtractionDir.toPath().resolve("elasticsearch-${VersionProperties.elasticsearch}/LICENSE.txt")
+      final List<String> actualLines = Files.readAllLines(licensePath)
+      assertLinesInFile(licensePath, actualLines, licenseLines)
     }
-  } else { // tar
-    testExtraction {
-      onlyIf { new File('/bin/tar').exists() || new File('/usr/bin/tar').exists() || new File('/usr/local/bin/tar').exists() }
-      commandLine 'tar', '-xvzf', "${-> buildDist.outputs.files.singleFile}", '-C', extractionDir
+  }
+  check.dependsOn checkLicense
+
+  task checkNotice {
+    dependsOn buildDist, checkExtraction
+    onlyIf toolExists
+    doLast {
+      final List<String> noticeLines = Arrays.asList("Elasticsearch", "Copyright 2009-2018 Elasticsearch")
+      final Path noticePath = archiveExtractionDir.toPath().resolve("elasticsearch-${VersionProperties.elasticsearch}/NOTICE.txt")
+      final List<String> actualLines = Files.readAllLines(noticePath)
+      assertLinesInFile(noticePath, actualLines, noticeLines)
     }
   }
-  check.dependsOn testExtraction
+  check.dependsOn checkNotice
+
 }
 
 /*****************************************************************************

distribution/build.gradle

@@ -17,16 +17,12 @@
  * under the License.
  */
 
-import org.apache.tools.ant.filters.FixCrLfFilter
-import org.apache.tools.ant.taskdefs.condition.Os
-import org.elasticsearch.gradle.BuildPlugin
 import org.elasticsearch.gradle.ConcatFilesTask
 import org.elasticsearch.gradle.MavenFilteringHack
 import org.elasticsearch.gradle.NoticeTask
-import org.elasticsearch.gradle.precommit.DependencyLicensesTask
-import org.elasticsearch.gradle.precommit.UpdateShasTask
 import org.elasticsearch.gradle.test.RunTask
-import org.gradle.api.file.RelativePath
+
+import java.nio.file.Path
 
 Collection distributions = project('archives').subprojects + project('packages').subprojects
 
@@ -464,3 +460,14 @@ subprojects {
     return result
   }
 }
+
+static void assertLinesInFile(final Path path, final List<String> actualLines, final List<String> expectedLines) {
+  int line = 0
+  for (final String expectedLine : expectedLines) {
+    final String actualLine = actualLines.get(line)
+    if (expectedLine != actualLine) {
+      throw new GradleException("expected line [${line + 1}] in [${path}] to be [${expectedLine}] but was [${actualLine}]")
+    }
+    line++
+  }
+}

distribution/packages/build.gradle

@@ -15,9 +15,15 @@
  * KIND, either express or implied.  See the License for the
  */
 
+
 import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.MavenFilteringHack
 
+import java.nio.file.Files
+import java.nio.file.Path
+import java.util.regex.Matcher
+import java.util.regex.Pattern
+
 /*****************************************************************************
  *                         Deb and rpm configuration                         *
  *****************************************************************************
@@ -117,19 +123,6 @@ Closure commonPackageConfig(String type, boolean oss) {
       from(rootProject.projectDir) {
         include 'README.textile'
       }
-      // Deb gets a copyright file instead.
-      if (type == 'rpm') {
-        from(rootProject.file('licenses')) {
-          include oss ? 'APACHE-LICENSE-2.0.txt' : 'ELASTIC-LICENSE.txt'
-          rename { 'LICENSE.txt' }
-        }
-      } else {
-        assert type == 'deb'
-        into("/usr/share/doc/${packageName}") {
-          from "${packagingFiles}/copyright"
-          fileMode 0644
-        }
-      }
       into('modules') {
         with copySpec {
           with modulesFiles(oss)
@@ -147,6 +140,22 @@ Closure commonPackageConfig(String type, boolean oss) {
       }
     }
 
+    // license files
+    if (type == 'deb') {
+      into("/usr/share/doc/${packageName}") {
+        from "${packagingFiles}/copyright"
+        fileMode 0644
+      }
+    } else {
+      assert type == 'rpm'
+      into('/usr/share/elasticsearch') {
+        from(rootProject.file('licenses')) {
+          include oss ? 'APACHE-LICENSE-2.0.txt' : 'ELASTIC-LICENSE.txt'
+          rename { 'LICENSE.txt' }
+        }
+      }
+    }
+
     // ========= config files =========
     configurationFile '/etc/elasticsearch/elasticsearch.yml'
     configurationFile '/etc/elasticsearch/jvm.options'
@@ -323,6 +332,11 @@ task buildOssRpm(type: Rpm) {
   configure(commonRpmConfig(true))
 }
 
+Closure dpkgExists = { it -> new File('/bin/dpkg-deb').exists() || new File('/usr/bin/dpkg-deb').exists() || new File('/usr/local/bin/dpkg-deb').exists() }
+Closure rpmExists = { it -> new File('/bin/rpm').exists() || new File('/usr/bin/rpm').exists() || new File('/usr/local/bin/rpm').exists() }
+
+Closure debFilter = { f -> f.name.endsWith('.deb') }
+
 // This configures the default artifact for the distribution specific
 // subprojects. We have subprojects because Gradle project substitutions
 // can only bind to the default configuration of a project
@@ -335,26 +349,33 @@ subprojects {
     'default' buildDist
   }
 
-  // sanity checks if a archives can be extracted
-  File extractionDir = new File(buildDir, 'extracted')
-  task testExtraction(type: LoggedExec) {
+  // sanity checks if packages can be extracted
+  final File extractionDir = new File(buildDir, 'extracted')
+  final File packageExtractionDir
+  if (project.name.contains('deb')) {
+    packageExtractionDir = new File(extractionDir, 'deb-extracted')
+  } else {
+    assert project.name.contains('rpm')
+    packageExtractionDir = new File(extractionDir, 'rpm-extracted')
+  }
+  task checkExtraction(type: LoggedExec) {
     dependsOn buildDist
     doFirst {
       project.delete(extractionDir)
       extractionDir.mkdirs()
     }
   }
+  check.dependsOn checkExtraction
   if (project.name.contains('deb')) {
-    testExtraction {
-      onlyIf { new File('/bin/dpkg-deb').exists() || new File('/usr/bin/dpkg-deb').exists() || new File('/usr/local/bin/dpkg-deb').exists() }
-      Closure debFilter = { f -> f.name.endsWith('.deb') }
-      commandLine 'dpkg-deb', '-x', "${-> buildDist.outputs.files.filter(debFilter).singleFile}", extractionDir
+    checkExtraction {
+      onlyIf dpkgExists
+      commandLine 'dpkg-deb', '-x', "${-> buildDist.outputs.files.filter(debFilter).singleFile}", packageExtractionDir
     }
-  } else { // rpm
-    testExtraction {
-      onlyIf { new File('/bin/rpm').exists() || new File('/usr/bin/rpm').exists() || new File('/usr/local/bin/rpm').exists() }
+  } else {
+    assert project.name.contains('rpm')
+    checkExtraction {
+      onlyIf rpmExists
       final File rpmDatabase = new File(extractionDir, 'rpm-database')
-      final File rpmExtracted = new File(extractionDir, 'rpm-extracted')
       commandLine 'rpm',
           '--badreloc',
           '--nodeps',
@@ -363,10 +384,131 @@ subprojects {
           '--dbpath',
           rpmDatabase,
           '--relocate',
-          "/=${rpmExtracted}",
+          "/=${packageExtractionDir}",
           '-i',
           "${-> buildDist.outputs.files.singleFile}"
     }
   }
-  check.dependsOn testExtraction
+
+  task checkLicense {
+    dependsOn buildDist, checkExtraction
+  }
+  check.dependsOn checkLicense
+  if (project.name.contains('deb')) {
+    checkLicense {
+      onlyIf dpkgExists
+      doLast {
+        final Path copyrightPath
+        final String expectedLicense
+        final String licenseFilename
+        if (project.name.contains('oss-')) {
+          copyrightPath = packageExtractionDir.toPath().resolve("usr/share/doc/elasticsearch-oss/copyright")
+          expectedLicense = "ASL-2.0"
+          licenseFilename = "APACHE-LICENSE-2.0.txt"
+        } else {
+          copyrightPath = packageExtractionDir.toPath().resolve("usr/share/doc/elasticsearch/copyright")
+          expectedLicense = "Elastic-License"
+          licenseFilename = "ELASTIC-LICENSE.txt"
+        }
+        final List<String> header = Arrays.asList("Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/",
+                "Copyright: Elasticsearch B.V. <info@elastic.co>",
+                "License: " + expectedLicense)
+        final List<String> licenseLines = Files.readAllLines(rootDir.toPath().resolve("licenses/" + licenseFilename))
+        final List<String> expectedLines = header + licenseLines.collect { " " + it }
+        final List<String> actualLines = Files.readAllLines(copyrightPath)
+        assertLinesInFile(copyrightPath, actualLines, expectedLines)
+      }
+    }
+  } else {
+    assert project.name.contains('rpm')
+    checkLicense {
+      onlyIf rpmExists
+      doLast {
+        final String licenseFilename
+        if (project.name.contains('oss-')) {
+          licenseFilename = "APACHE-LICENSE-2.0.txt"
+        } else {
+          licenseFilename = "ELASTIC-LICENSE.txt"
+        }
+        final List<String> licenseLines = Files.readAllLines(rootDir.toPath().resolve("licenses/" + licenseFilename))
+        final Path licensePath = packageExtractionDir.toPath().resolve("usr/share/elasticsearch/LICENSE.txt")
+        final List<String> actualLines = Files.readAllLines(licensePath)
+        assertLinesInFile(licensePath, actualLines, licenseLines)
+      }
+    }
+  }
+
+  task checkNotice {
+    dependsOn buildDist, checkExtraction
+    onlyIf { (project.name.contains('deb') && dpkgExists.call(it)) || (project.name.contains('rpm') && rpmExists.call(it)) }
+    doLast {
+      final List<String> noticeLines = Arrays.asList("Elasticsearch", "Copyright 2009-2018 Elasticsearch")
+      final Path noticePath = packageExtractionDir.toPath().resolve("usr/share/elasticsearch/NOTICE.txt")
+      final List<String> actualLines = Files.readAllLines(noticePath)
+      assertLinesInFile(noticePath, actualLines, noticeLines)
+    }
+  }
+  check.dependsOn checkNotice
+
+  task checkLicenseMetadata(type: LoggedExec) {
+    dependsOn buildDist, checkExtraction
+  }
+  check.dependsOn checkLicenseMetadata
+  if (project.name.contains('deb')) {
+    checkLicenseMetadata { LoggedExec exec ->
+      onlyIf dpkgExists
+      final ByteArrayOutputStream output = new ByteArrayOutputStream()
+      exec.commandLine 'dpkg-deb', '--info', "${ -> buildDist.outputs.files.filter(debFilter).singleFile}"
+      exec.standardOutput = output
+      doLast {
+        final String expectedLicense
+        if (project.name.contains('oss-')) {
+          expectedLicense = "ASL-2.0"
+        } else {
+          expectedLicense = "Elastic-License"
+        }
+        final Pattern pattern = Pattern.compile("\\s*License: (.+)")
+        final String info = output.toString('UTF-8')
+        final String[] actualLines = info.split("\n")
+        int count = 0
+        for (final String actualLine : actualLines) {
+          final Matcher matcher = pattern.matcher(actualLine)
+          if (matcher.matches()) {
+            count++
+            final String actualLicense = matcher.group(1)
+            if (expectedLicense != actualLicense) {
+              throw new GradleException("expected license [${expectedLicense} for package info but found [${actualLicense}]")
+            }
+          }
+        }
+        if (count == 0) {
+          throw new GradleException("expected license [${expectedLicense}] for package info but found none in:\n${info}")
+        }
+        if (count > 1) {
+          throw new GradleException("expected a single license for package info but found [${count}] in:\n${info}")
+        }
+      }
+    }
+  } else {
+    assert project.name.contains('rpm')
+    checkLicenseMetadata { LoggedExec exec ->
+      onlyIf rpmExists
+      final ByteArrayOutputStream output = new ByteArrayOutputStream()
+      exec.commandLine 'rpm', '-qp', '--queryformat', '%{License}', "${-> buildDist.outputs.files.singleFile}"
+      exec.standardOutput = output
+      doLast {
+        final String license = output.toString('UTF-8')
+        final String expectedLicense
+        if (project.name.contains('oss-')) {
+          expectedLicense = "ASL 2.0"
+        } else {
+          expectedLicense = "Elastic License"
+        }
+        if (license != expectedLicense) {
+          throw new GradleException("expected license [${expectedLicense}] for [${-> buildDist.outputs.files.singleFile}] but was [${license}]")
+        }
+      }
+    }
+  }
+
 }