Build: Automatically add projects under libs, qa, modules and plugins (#28279)

This commit lessens the burden on configuring settings.gradle when new
projects are added. In particular, this makes it trivial to move a
plugin to a module (or vice versa).

settings.gradle

@@ -27,78 +27,56 @@ List projects = [
   'test:fixtures:hdfs-fixture',
   'test:fixtures:krb5kdc-fixture',
   'test:fixtures:old-elasticsearch',
-  'test:logger-usage',
-  'libs:elasticsearch-core',
-  'libs:elasticsearch-nio',
-  'modules:aggs-matrix-stats',
-  'modules:analysis-common',
-  'modules:ingest-common',
-  'modules:lang-expression',
-  'modules:lang-mustache',
-  'modules:lang-painless',
-  'modules:mapper-extras',
-  'modules:parent-join',
-  'modules:percolator',
-  'modules:rank-eval',
-  'modules:reindex',
-  'modules:repository-url',
-  'modules:transport-netty4',
-  'modules:tribe',
-  'plugins:analysis-icu',
-  'plugins:analysis-kuromoji',
-  'plugins:analysis-phonetic',
-  'plugins:analysis-smartcn',
-  'plugins:analysis-stempel',
-  'plugins:analysis-ukrainian',
-  'plugins:discovery-azure-classic',
-  'plugins:discovery-ec2',
-  'plugins:discovery-file',
-  'plugins:discovery-gce',
-  'plugins:ingest-geoip',
-  'plugins:ingest-attachment',
-  'plugins:ingest-user-agent',
-  'plugins:mapper-murmur3',
-  'plugins:mapper-size',
-  'plugins:repository-azure',
-  'plugins:repository-gcs',
-  'plugins:repository-hdfs',
-  'plugins:repository-s3',
-  'plugins:jvm-example',
-  'plugins:store-smb',
-  'plugins:transport-nio',
-  'qa:auto-create-index',
-  'qa:ccs-unavailable-clusters',
-  'qa:evil-tests',
-  'qa:full-cluster-restart',
-  'qa:integration-bwc',
-  'qa:mixed-cluster',
-  'qa:multi-cluster-search',
-  'qa:no-bootstrap-tests',
-  'qa:reindex-from-old',
-  'qa:rolling-upgrade',
-  'qa:smoke-test-client',
-  'qa:smoke-test-http',
-  'qa:smoke-test-ingest-with-all-dependencies',
-  'qa:smoke-test-ingest-disabled',
-  'qa:smoke-test-multinode',
-  'qa:smoke-test-rank-eval-with-mustache',
-  'qa:smoke-test-plugins',
-  'qa:smoke-test-reindex-with-all-modules',
-  'qa:smoke-test-tribe-node',
-  'qa:vagrant',
-  'qa:verify-version-constants',
-  'qa:wildfly',
-  'qa:query-builder-bwc'
+  'test:logger-usage'
 ]
 
-projects.add("libs")
-File libsDir = new File(rootProject.projectDir, 'libs')
-for (File libDir : new File(rootProject.projectDir, 'libs').listFiles()) {
-  if (libDir.isDirectory() == false) continue;
-  if (libDir.name.startsWith('build') || libDir.name.startsWith('.')) continue;
-  projects.add("libs:${libDir.name}".toString())
+/**
+  * Iterates over sub directories, looking for build.gradle, and adds a project if found
+  * for that dir with the given path prefix. Note that this requires each level
+  * of the dir hierarchy to have a build.gradle. Otherwise we would have to iterate
+  * all files/directories in the source tree to find all projects.
+  */
+void addSubProjects(String path, File dir, List<String> projects, List<String> branches) {
+  if (dir.isDirectory() == false) return;
+  if (dir.name == 'buildSrc') return;
+  if (new File(dir, 'build.gradle').exists() == false) return;
+  if (findProject(dir) != null) return;
+
+  final String projectName = "${path}:${dir.name}"
+  include projectName
+
+  if (dir.name == 'bwc-snapshot-dummy-projects') {
+    for (final String branch : branches) {
+      final String snapshotProjectName = "${projectName}:bwc-snapshot-${branch}"
+      projects.add(snapshotProjectName)
+      include snapshotProjectName
+      project("${snapshotProjectName}").projectDir = dir
+    }
+    // TODO do we want to assert that there's nothing else in the bwc directory?
+  } else {
+    if (path.isEmpty() || path.startsWith(':example-plugins')) {
+      project(projectName).projectDir = dir
+    }
+    for (File subdir : dir.listFiles()) {
+      addSubProjects(projectName, subdir, projects, branches)
+    }
+  }
 }
 
+// include example plugins first, so adding plugin dirs below won't muck with :example-plugins
+File examplePluginsDir = new File(rootProject.projectDir, 'plugins/examples')
+for (File example : examplePluginsDir.listFiles()) {
+  if (example.isDirectory() == false) continue;
+  if (example.name.startsWith('build') || example.name.startsWith('.')) continue;
+  addSubProjects(':example-plugins', example, projects, [])
+}
+project(':example-plugins').projectDir = new File(rootProject.projectDir, 'plugins/examples')
+
+addSubProjects('', new File(rootProject.projectDir, 'libs'), projects, [])
+addSubProjects('', new File(rootProject.projectDir, 'modules'), projects, [])
+addSubProjects('', new File(rootProject.projectDir, 'plugins'), projects, [])
+addSubProjects('', new File(rootProject.projectDir, 'qa'), projects, [])
+
 /* Create projects for building BWC snapshot distributions from the heads of other branches */
 final List<String> branches = ['5.6', '6.0', '6.1', '6.x']
 for (final String branch : branches) {
@@ -139,47 +117,6 @@ if (isEclipse) {
   project(":libs:elasticsearch-nio-tests").buildFileName = 'eclipse-build.gradle'
 }
 
-/**
-  * Iterates over sub directories, looking for build.gradle, and adds a project if found
-  * for that dir with the given path prefix. Note that this requires each level
-  * of the dir hierarchy to have a build.gradle. Otherwise we would have to iterate
-  * all files/directories in the source tree to find all projects.
-  */
-void addSubProjects(String path, File dir, List<String> projects, List<String> branches) {
-  if (dir.isDirectory() == false) return;
-  if (dir.name == 'buildSrc') return;
-  if (new File(dir, 'build.gradle').exists() == false) return;
-
-  final String projectName = "${path}:${dir.name}"
-  include projectName
-
-  if (dir.name == 'bwc-snapshot-dummy-projects') {
-    for (final String branch : branches) {
-      final String snapshotProjectName = "${projectName}:bwc-snapshot-${branch}"
-      projects.add(snapshotProjectName)
-      include snapshotProjectName
-      project("${snapshotProjectName}").projectDir = dir
-    }
-    // TODO do we want to assert that there's nothing else in the bwc directory?
-  } else {
-    if (path.isEmpty() || path.startsWith(':example-plugins')) {
-      project(projectName).projectDir = dir
-    }
-    for (File subdir : dir.listFiles()) {
-      addSubProjects(projectName, subdir, projects, branches)
-    }
-  }
-}
-
-// include example plugins
-File examplePluginsDir = new File(rootProject.projectDir, 'plugins/examples')
-for (File example : examplePluginsDir.listFiles()) {
-  if (example.isDirectory() == false) continue;
-  if (example.name.startsWith('build') || example.name.startsWith('.')) continue;
-  addSubProjects(':example-plugins', example, projects, [])
-}
-project(':example-plugins').projectDir = new File(rootProject.projectDir, 'plugins/examples')
-
 // look for extra plugins for elasticsearch
 File extraProjects = new File(rootProject.projectDir.parentFile, "${dirName}-extra")
 if (extraProjects.exists()) {
@@ -187,5 +124,4 @@ if (extraProjects.exists()) {
     addSubProjects('', extraProjectDir, projects, branches)
   }
 }
-include 'libs'