Merge branch 'master' into staging_plugins

build.gradle

@@ -269,13 +269,6 @@ tasks.idea.doLast {
 if (System.getProperty('idea.active') != null && ideaMarker.exists() == false) {
   throw new GradleException('You must run gradle idea from the root of elasticsearch before importing into IntelliJ')
 }
-// add buildSrc itself as a groovy project
-task buildSrcIdea(type: GradleBuild) {
-  buildFile = 'buildSrc/build.gradle'
-  tasks = ['cleanIdea', 'ideaModule']
-}
-tasks.idea.dependsOn(buildSrcIdea)
-
 
 // eclipse configuration
 allprojects {
@@ -318,13 +311,6 @@ allprojects {
   tasks.eclipse.dependsOn(cleanEclipse, copyEclipseSettings)
 }
 
-// add buildSrc itself as a groovy project
-task buildSrcEclipse(type: GradleBuild) {
-  buildFile = 'buildSrc/build.gradle'
-  tasks = ['cleanEclipse', 'eclipse']
-}
-tasks.eclipse.dependsOn(buildSrcEclipse)
-
 // we need to add the same --debug-jvm option as
 // the real RunTask has, so we can pass it through
 class Run extends DefaultTask {

core/build.gradle

@@ -121,6 +121,36 @@ forbiddenPatterns {
   exclude '**/org/elasticsearch/cluster/routing/shard_routes.txt'
 }
 
+task generateModulesList {
+  List<String> modules = project(':modules').subprojects.collect { it.name }
+  File modulesFile = new File(buildDir, 'generated-resources/modules.txt')
+  processResources.from(modulesFile) 
+  inputs.property('modules', modules) 
+  outputs.file(modulesFile)
+  doLast {
+    modulesFile.parentFile.mkdirs()
+    modulesFile.setText(modules.join('\n'), 'UTF-8')
+  }
+}
+
+task generatePluginsList {
+  List<String> plugins = project(':plugins').subprojects
+    .findAll { it.name.contains('example') == false }
+    .collect { it.name }
+  File pluginsFile = new File(buildDir, 'generated-resources/plugins.txt')
+  processResources.from(pluginsFile) 
+  inputs.property('plugins', plugins) 
+  outputs.file(pluginsFile)
+  doLast {
+    pluginsFile.parentFile.mkdirs()
+    pluginsFile.setText(plugins.join('\n'), 'UTF-8')
+  }
+}
+
+processResources {
+  dependsOn generateModulesList, generatePluginsList
+}
+
 thirdPartyAudit.excludes = [
   // uses internal java api: sun.security.x509 (X509CertInfo, X509CertImpl, X500Name)
   'org.jboss.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator',

core/src/main/java/org/elasticsearch/Version.java

@@ -32,7 +32,6 @@ import java.io.IOException;
 
 /**
  */
-@SuppressWarnings("deprecation")
 public class Version {
     /*
      * The logic for ID is: XXYYZZAA, where XX is major version, YY is minor version, ZZ is revision, and AA is alpha/beta/rc indicator AA

core/src/main/java/org/elasticsearch/index/analysis/StopTokenFilterFactory.java

@@ -34,7 +34,6 @@ import java.util.Set;
 /**
  *
  */
-@SuppressWarnings("deprecation")
 public class StopTokenFilterFactory extends AbstractTokenFilterFactory {
 
     private final CharArraySet stopWords;

core/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java

@@ -129,7 +129,6 @@ public class FsDirectoryService extends DirectoryService implements StoreRateLim
         throw new IllegalArgumentException("No directory found for type [" + storeType + "]");
     }
 
-    @SuppressWarnings("deprecation")
     private static boolean isDefault(String storeType) {
         return IndexModule.Type.DEFAULT.match(storeType);
     }

core/src/main/java/org/elasticsearch/index/store/Store.java

@@ -821,7 +821,6 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
                 final SegmentInfos segmentCommitInfos = Store.readSegmentsInfo(commit, directory);
                 numDocs = Lucene.getNumDocs(segmentCommitInfos);
                 commitUserDataBuilder.putAll(segmentCommitInfos.getUserData());
-                @SuppressWarnings("deprecation")
                 Version maxVersion = segmentCommitInfos.getMinSegmentLuceneVersion(); // we don't know which version was used to write so we take the max version.
                 for (SegmentCommitInfo info : segmentCommitInfos) {
                     final Version version = info.info.getVersion();

core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java

@@ -19,22 +19,6 @@
 
 package org.elasticsearch.plugins;
 
-import joptsimple.OptionSet;
-import joptsimple.OptionSpec;
-import org.apache.lucene.util.IOUtils;
-import org.elasticsearch.Build;
-import org.elasticsearch.Version;
-import org.elasticsearch.bootstrap.JarHell;
-import org.elasticsearch.cli.ExitCodes;
-import org.elasticsearch.cli.SettingCommand;
-import org.elasticsearch.cli.Terminal;
-import org.elasticsearch.cli.UserError;
-import org.elasticsearch.common.hash.MessageDigests;
-import org.elasticsearch.common.io.FileSystemUtils;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.node.internal.InternalSettingsPreparer;
-
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
@@ -55,18 +39,31 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
-import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.TreeSet;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 
-import static java.util.Collections.unmodifiableSet;
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.Version;
+import org.elasticsearch.bootstrap.JarHell;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.SettingCommand;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.UserError;
+import org.elasticsearch.common.hash.MessageDigests;
+import org.elasticsearch.common.io.FileSystemUtils;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.node.internal.InternalSettingsPreparer;
+
 import static org.elasticsearch.cli.Terminal.Verbosity.VERBOSE;
-import static org.elasticsearch.common.util.set.Sets.newHashSet;
 
 /**
  * A command for the plugin cli to install a plugin into elasticsearch.
@@ -102,37 +99,40 @@ class InstallPluginCommand extends SettingCommand {
 
     private static final String PROPERTY_STAGING_ID = "es.plugins.staging";
 
-    // TODO: make this a resource file generated by gradle
-    static final Set<String> MODULES = unmodifiableSet(newHashSet(
-            "ingest-grok",
-            "lang-expression",
-            "lang-groovy",
-            "lang-painless",
-            "reindex"));
-
-    // TODO: make this a resource file generated by gradle
-    static final Set<String> OFFICIAL_PLUGINS = unmodifiableSet(new LinkedHashSet<>(Arrays.asList(
-            "analysis-icu",
-            "analysis-kuromoji",
-            "analysis-phonetic",
-            "analysis-smartcn",
-            "analysis-stempel",
-            "discovery-azure",
-            "discovery-ec2",
-            "discovery-gce",
-            "ingest-attachment",
-            "ingest-geoip",
-            "lang-javascript",
-            "lang-python",
-            "mapper-attachments",
-            "mapper-murmur3",
-            "mapper-size",
-            "repository-azure",
-            "repository-gcs",
-            "repository-hdfs",
-            "repository-s3",
-            "store-smb",
-            "x-pack")));
+    /** The builtin modules, which are plugins, but cannot be installed or removed. */
+    static final Set<String> MODULES;
+    static {
+        try (InputStream stream = InstallPluginCommand.class.getResourceAsStream("/modules.txt");
+             BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) {
+            Set<String> modules = new HashSet<>();
+            String line = reader.readLine();
+            while (line != null) {
+                modules.add(line.trim());
+                line = reader.readLine();
+            }
+            MODULES = Collections.unmodifiableSet(modules);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /** The official plugins that can be installed simply by name. */
+    static final Set<String> OFFICIAL_PLUGINS;
+    static {
+        try (InputStream stream = InstallPluginCommand.class.getResourceAsStream("/plugins.txt");
+             BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) {
+            Set<String> plugins = new TreeSet<>(); // use tree set to get sorting for help command
+            String line = reader.readLine();
+            while (line != null) {
+                plugins.add(line.trim());
+                line = reader.readLine();
+            }
+            plugins.add("x-pack");
+            OFFICIAL_PLUGINS = Collections.unmodifiableSet(plugins);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
 
     private final OptionSpec<Void> batchOption;
     private final OptionSpec<String> arguments;

core/src/test/java/org/elasticsearch/index/query/QueryDSLDocumentationTests.java

@@ -138,7 +138,6 @@ public class QueryDSLDocumentationTests extends ESTestCase {
         functionScoreQuery(functions);
     }
 
-    @SuppressWarnings("deprecation") // fuzzy queries will be removed in 4.0
     public void testFuzzy() {
         fuzzyQuery("name", "kimchy");
     }

core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java

@@ -2269,7 +2269,6 @@ public class HighlighterSearchIT extends ESIntegTestCase {
 
     }
 
-    @SuppressWarnings("deprecation") // fuzzy queries will be removed in 4.0
     public void testPostingsHighlighterFuzzyQuery() throws Exception {
         assertAcked(prepareCreate("test").addMapping("type1", type1PostingsffsetsMapping()));
         ensureGreen();

core/src/test/java/org/elasticsearch/search/matchedqueries/MatchedQueriesIT.java

@@ -252,7 +252,6 @@ public class MatchedQueriesIT extends ESIntegTestCase {
         }
     }
 
-    @SuppressWarnings("deprecation") // fuzzy queries will be removed in 4.0
     public void testFuzzyQuerySupportsName() {
         createIndex("test1");
         ensureGreen();

core/src/test/java/org/elasticsearch/search/profile/query/RandomQueryGenerator.java

@@ -72,7 +72,6 @@ public class RandomQueryGenerator {
         }
     }
 
-    @SuppressWarnings("deprecation") // fuzzy queries will be removed in 4.0
     private static QueryBuilder randomTerminalQuery(List<String> stringFields, List<String> numericFields, int numDocs) {
         switch (randomIntBetween(0,6)) {
             case 0:
@@ -196,7 +195,6 @@ public class RandomQueryGenerator {
         return q;
     }
 
-    @SuppressWarnings("deprecation") // fuzzy queries will be removed in 4.0
     @Deprecated
     private static QueryBuilder randomFuzzyQuery(List<String> fields) {
 

core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java

@@ -1440,7 +1440,6 @@ public class SearchQueryIT extends ESIntegTestCase {
         assertHitCount(searchResponse, 3L);
     }
 
-    @SuppressWarnings("deprecation") // fuzzy queries will be removed in 4.0
     public void testSpanMultiTermQuery() throws IOException {
         createIndex("test");
 

core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java

@@ -185,7 +185,6 @@ public class GeoDistanceSortBuilderIT extends ESIntegTestCase {
         builder.endObject();
     }
 
-    @SuppressWarnings("deprecation")
     public void testManyToManyGeoPointsWithDifferentFormats() throws ExecutionException, InterruptedException, IOException {
         /**   q     d1       d2
          * |4  o|   x    |   x

core/src/test/java/org/elasticsearch/validate/SimpleValidateQueryIT.java

@@ -212,7 +212,6 @@ public class SimpleValidateQueryIT extends ESIntegTestCase {
         assertThat(validateQueryResponse.getQueryExplanation().get(0).getExplanation(), containsString("field:\"foo (one* two*)\""));
     }
 
-    @SuppressWarnings("deprecation") // fuzzy queries will be removed in 4.0
     public void testExplainWithRewriteValidateQuery() throws Exception {
         client().admin().indices().prepareCreate("test")
                 .addMapping("type1", "field", "type=text,analyzer=whitespace")

plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuCollationTokenFilterFactory.java

@@ -46,7 +46,6 @@ public class IcuCollationTokenFilterFactory extends AbstractTokenFilterFactory {
 
     private final Collator collator;
 
-    @SuppressWarnings("deprecation") // Intentionally sets deprecated options for backwards compatibility
     public IcuCollationTokenFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
         super(indexSettings, name, settings);
 
@@ -167,7 +166,6 @@ public class IcuCollationTokenFilterFactory extends AbstractTokenFilterFactory {
     }
 
     @Override
-    @SuppressWarnings("deprecation") // Constructs a deprecated filter for backwards compatibility
     public TokenStream create(TokenStream tokenStream) {
         return new ICUCollationKeyFilter(tokenStream, collator);
     }

plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/AttachmentMapper.java

@@ -427,7 +427,6 @@ public class AttachmentMapper extends FieldMapper {
     }
 
     @Override
-    @SuppressWarnings("deprecation") // https://github.com/elastic/elasticsearch/issues/15843
     public Mapper parse(ParseContext context) throws IOException {
         byte[] content = null;
         String contentType = null;

qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java

@@ -36,8 +36,10 @@ import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.PosixPermissionsResetter;
 import org.junit.After;
 
+import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.StringReader;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
@@ -532,6 +534,34 @@ public class InstallPluginCommandTests extends ESTestCase {
         assertTrue(e.getMessage(), e.getMessage().contains("resolving outside of plugin directory"));
     }
 
+    public void testOfficialPluginsHelpSorted() throws Exception {
+        MockTerminal terminal = new MockTerminal();
+        new InstallPluginCommand().main(new String[] { "--help" }, terminal);
+        try (BufferedReader reader = new BufferedReader(new StringReader(terminal.getOutput()))) {
+            String line = reader.readLine();
+
+            // first find the beginning of our list of official plugins
+            while (line.endsWith("may be installed by name:") == false) {
+                line = reader.readLine();
+            }
+
+            // now check each line compares greater than the last, until we reach an empty line
+            String prev = reader.readLine();
+            line = reader.readLine();
+            while (line != null && line.trim().isEmpty() == false) {
+                assertTrue(prev + " < " + line, prev.compareTo(line) < 0);
+                prev = line;
+                line = reader.readLine();
+            }
+        }
+    }
+
+    public void testOfficialPluginsIncludesXpack() throws Exception {
+        MockTerminal terminal = new MockTerminal();
+        new InstallPluginCommand().main(new String[] { "--help" }, terminal);
+        assertTrue(terminal.getOutput(), terminal.getOutput().contains("x-pack"));
+    }
+
     // TODO: test batch flag?
     // TODO: test checksum (need maven/official below)
     // TODO: test maven, official, and staging install...need tests with fixtures...

test/framework/src/main/java/org/elasticsearch/common/io/PathUtilsForTesting.java

@@ -29,7 +29,6 @@ import java.nio.file.FileSystem;
 public class PathUtilsForTesting {
 
     /** Sets a new default filesystem for testing */
-    @SuppressWarnings("deprecation") // https://github.com/elastic/elasticsearch/issues/15845
     public static void setup() {
         installMock(LuceneTestCase.getBaseTempDirForTestClass().getFileSystem());
     }

test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java

@@ -1033,7 +1033,6 @@ public final class InternalTestCluster extends TestCluster {
         }
     }
 
-    @SuppressWarnings("deprecation") // https://github.com/elastic/elasticsearch/issues/15844
     private void randomlyResetClients() throws IOException {
         // only reset the clients on nightly tests, it causes heavy load...
         if (RandomizedTest.isNightly() && rarely(random)) {

test/framework/src/main/java/org/elasticsearch/test/store/MockFSDirectoryService.java

@@ -113,7 +113,6 @@ public class MockFSDirectoryService extends FsDirectoryService {
         throw new UnsupportedOperationException();
     }
 
-    @SuppressWarnings("deprecation") // https://github.com/elastic/elasticsearch/issues/15846
     public static void checkIndex(ESLogger logger, Store store, ShardId shardId) {
         if (store.tryIncRef()) {
             logger.info("start check index");