[TEST] Expose `tests.filter` for elasticsearch tests.

`-Dtests.filter` allows to pass filter expressions to the elasticsearch
tests. This allows to filter test annotaged with TestGroup annotations
like @Slow, @Nightly, @Backwards, @Integration with a boolean expresssion like:

 * to run only backwards tests run:
     `mvn -Dtests.bwc.version=X.Y.Z -Dtests.filter="@backwards"`
 * to run all integration tests but skip slow tests run:
     `mvn -Dtests.filter="@integration and not @slow"
 * to take defaults into account ie run all test as well as backwards:
     `mvn -Dtests.filter="default and @backwards"

This feature is a more powerful alternative to flags like
`-Dtests.nighly=true|false` etc.

Closes #6703

TESTING.asciidoc

@@ -62,6 +62,29 @@ Run any test methods that contain 'esi' (like: ...r*esi*ze...).
 mvn test "-Dtests.method=*esi*"
 -------------------------------
 
+You can also filter tests by certain annotations ie:
+
+  * `@Slow` - tests that are know to take a long time to execute
+  * `@Nightly` - tests that only run in nightly builds (disabled by default)
+  * `@Integration` - integration tests
+  * `@Backwards` - backwards compatibility tests (disabled by default)
+  * `@AwaitsFix` - tests that are waiting for a bugfix (disabled by default)
+  * `@BadApple` - tests that are known to fail randomly (disabled by default)
+
+Those annotation names can be combined into a filter expression like:
+
+------------------------------------------------
+mvn test -Dtests.filter="@nightly and not @slow" 
+------------------------------------------------
+
+to run all nightly test but not the ones that are slow. `tests.filter` supports
+the boolean operators `and, or, not` and grouping ie:
+
+
+---------------------------------------------------------------
+mvn test -Dtests.filter="@nightly and not(@slow or @backwards)" 
+---------------------------------------------------------------
+
 === Seed and repetitions.
 
 Run with a given seed (seed is a hex-encoded long).
@@ -184,14 +207,14 @@ To run backwards compatibiilty tests untar or unzip a release and run the tests
 with the following command:
 
 ---------------------------------------------------------------------------
-mvn test -Dtests.bwc=true -Dtests.bwc.version=x.y.z -Dtests.bwc.path=/path/to/elasticsearch 
+mvn test -Dtests.filter="@backwards" -Dtests.bwc.version=x.y.z -Dtests.bwc.path=/path/to/elasticsearch
 ---------------------------------------------------------------------------
 
 If the elasticsearch release is placed under `./backwards/elasticsearch-x.y.z` the path
 can be omitted:
  
 ---------------------------------------------------------------------------
-mvn test -Dtests.bwc=true -Dtests.bwc.version=x.y.z 
+mvn test -Dtests.filter="@backwards" -Dtests.bwc.version=x.y.z
 ---------------------------------------------------------------------------
 
 To setup the bwc test environment execute the following steps (provided you are

pom.xml

@@ -501,6 +501,7 @@
                                 <tests.network>${tests.network}</tests.network>
                                 <tests.cluster>${tests.cluster}</tests.cluster>
                                 <tests.heap.size>${tests.heap.size}</tests.heap.size>
+                                <tests.filter>${tests.filter}</tests.filter>
                                 <es.node.local>${env.ES_TEST_LOCAL}</es.node.local>
                                 <es.node.mode>${es.node.mode}</es.node.mode>
                                 <es.logger.level>${es.logger.level}</es.logger.level>

src/test/java/org/apache/lucene/search/postingshighlight/CustomPassageFormatterTests.java

@@ -21,6 +21,7 @@ package org.apache.lucene.search.postingshighlight;
 import org.apache.lucene.search.highlight.DefaultEncoder;
 import org.apache.lucene.search.highlight.SimpleHTMLEncoder;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.test.ElasticsearchTestCase;
 import org.junit.Test;
 
 import static org.hamcrest.CoreMatchers.equalTo;
@@ -28,7 +29,7 @@ import static org.hamcrest.CoreMatchers.notNullValue;
 import static org.hamcrest.MatcherAssert.assertThat;
 
 
-public class CustomPassageFormatterTests {
+public class CustomPassageFormatterTests extends ElasticsearchTestCase {
 
     @Test
     public void testSimpleFormat() {

src/test/java/org/apache/lucene/util/AbstractRandomizedTest.java

@@ -77,12 +77,12 @@ public abstract class AbstractRandomizedTest extends RandomizedTest {
     @Retention(RetentionPolicy.RUNTIME)
     @Target(ElementType.TYPE)
     @TestGroup(enabled = false, sysProperty = TESTS_BACKWARDS_COMPATIBILITY)
-    public @interface BackwardsCompatibilityTest {
+    public @interface Backwards {
     }
 
     /**
      * Key used to set the path for the elasticsearch executable used to run backwards compatibility tests from
-     * via the commandline -D{@value #TESTS_BACKWARDS_COMPATIBILITY_PATH}
+     * via the commandline -D{@value #TESTS_BACKWARDS_COMPATIBILITY}
      */
     public static final String TESTS_BACKWARDS_COMPATIBILITY = "tests.bwc";
 
@@ -101,7 +101,7 @@ public abstract class AbstractRandomizedTest extends RandomizedTest {
     @Retention(RetentionPolicy.RUNTIME)
     @Target(ElementType.TYPE)
     @TestGroup(enabled = true, sysProperty = SYSPROP_INTEGRATION)
-    public @interface IntegrationTests {
+    public @interface Integration {
     }
 
     // --------------------------------------------------------------------

src/test/java/org/elasticsearch/common/lucene/LuceneTest.java

@@ -20,6 +20,7 @@ package org.elasticsearch.common.lucene;
 import org.apache.lucene.util.Version;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.ESLoggerFactory;
+import org.elasticsearch.test.ElasticsearchTestCase;
 import org.junit.Test;
 
 import static org.hamcrest.MatcherAssert.assertThat;
@@ -28,7 +29,7 @@ import static org.hamcrest.core.IsEqual.equalTo;
 /**
  * 
  */
-public class LuceneTest {
+public class LuceneTest extends ElasticsearchTestCase   {
 
 
     /*

src/test/java/org/elasticsearch/common/util/ArrayUtilsTests.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.common.util;
 
+import org.elasticsearch.test.ElasticsearchTestCase;
 import org.junit.Test;
 
 import java.util.Arrays;
@@ -30,7 +31,7 @@ import static org.hamcrest.Matchers.is;
 /**
  *
  */
-public class ArrayUtilsTests {
+public class ArrayUtilsTests extends ElasticsearchTestCase {
 
     @Test
     public void binarySearch() throws Exception {

src/test/java/org/elasticsearch/index/analysis/PreBuiltCharFilterFactoryFactoryTests.java

@@ -22,6 +22,7 @@ import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.indices.analysis.PreBuiltCharFilters;
+import org.elasticsearch.test.ElasticsearchTestCase;
 import org.junit.Test;
 
 import static org.hamcrest.CoreMatchers.*;
@@ -30,7 +31,7 @@ import static org.hamcrest.MatcherAssert.assertThat;
 /**
  *
  */
-public class PreBuiltCharFilterFactoryFactoryTests {
+public class PreBuiltCharFilterFactoryFactoryTests extends ElasticsearchTestCase {
 
     @Test
     public void testThatDifferentVersionsCanBeLoaded() {

src/test/java/org/elasticsearch/index/mapper/lucene/DoubleIndexingDocTest.java

@@ -24,12 +24,15 @@ import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.RAMDirectory;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.MapperTestUtils;
+import org.elasticsearch.test.ElasticsearchLuceneTestCase;
+import org.elasticsearch.test.ElasticsearchTestCase;
 import org.junit.Test;
 
 import static org.hamcrest.MatcherAssert.assertThat;
@@ -38,11 +41,12 @@ import static org.hamcrest.Matchers.equalTo;
 /**
  *
  */
-public class DoubleIndexingDocTest {
+public class DoubleIndexingDocTest extends ElasticsearchLuceneTestCase {
 
     @Test
     public void testDoubleIndexingSameDoc() throws Exception {
-        IndexWriter writer = new IndexWriter(new RAMDirectory(), new IndexWriterConfig(Lucene.VERSION, Lucene.STANDARD_ANALYZER));
+        Directory dir = newDirectory();
+        IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(random(), TEST_VERSION_CURRENT, Lucene.STANDARD_ANALYZER));
 
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("properties").endObject()
@@ -85,5 +89,8 @@ public class DoubleIndexingDocTest {
 
         topDocs = searcher.search(mapper.mappers().smartName("field5").mapper().termQuery("3", null), 10);
         assertThat(topDocs.totalHits, equalTo(2));
+        writer.close();
+        reader.close();
+        dir.close();
     }
 }

src/test/java/org/elasticsearch/index/mapper/lucene/StoredNumericValuesTest.java

@@ -33,6 +33,7 @@ import org.elasticsearch.index.fieldvisitor.CustomFieldsVisitor;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.MapperTestUtils;
+import org.elasticsearch.test.ElasticsearchTestCase;
 import org.junit.Test;
 
 import java.util.*;
@@ -43,7 +44,7 @@ import static org.hamcrest.Matchers.equalTo;
 /**
  *
  */
-public class StoredNumericValuesTest {
+public class StoredNumericValuesTest extends ElasticsearchTestCase{
 
     @Test
     public void testBytesAndNumericRepresentation() throws Exception {

src/test/java/org/elasticsearch/index/merge/policy/MergePolicySettingsTest.java

@@ -28,6 +28,7 @@ import org.elasticsearch.index.store.DirectoryService;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.index.store.distributor.LeastUsedDistributor;
 import org.elasticsearch.index.store.ram.RamDirectoryService;
+import org.elasticsearch.test.ElasticsearchTestCase;
 import org.junit.Test;
 
 import java.io.IOException;
@@ -35,8 +36,9 @@ import java.io.IOException;
 import static org.elasticsearch.common.settings.ImmutableSettings.Builder.EMPTY_SETTINGS;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
 
-public class MergePolicySettingsTest {
+public class MergePolicySettingsTest extends ElasticsearchTestCase {
 
     protected final ShardId shardId = new ShardId(new Index("index"), 1);
 
@@ -87,19 +89,19 @@ public class MergePolicySettingsTest {
         IndexSettingsService service = new IndexSettingsService(new Index("test"), EMPTY_SETTINGS);
         try {
             new LogDocMergePolicyProvider(createStore(build(-0.1)), service).newMergePolicy().getNoCFSRatio();
-            assertThat("exception expected", false);
+            fail("exception expected");
         } catch (ElasticsearchIllegalArgumentException ex) {
 
         }
         try {
             new LogDocMergePolicyProvider(createStore(build(1.1)), service).newMergePolicy().getNoCFSRatio();
-            assertThat("exception expected", false);
+            fail("exception expected");
         } catch (ElasticsearchIllegalArgumentException ex) {
 
         }
         try {
             new LogDocMergePolicyProvider(createStore(build("Falsch")), service).newMergePolicy().getNoCFSRatio();
-            assertThat("exception expected", false);
+            fail("exception expected");
         } catch (ElasticsearchIllegalArgumentException ex) {
 
         }

src/test/java/org/elasticsearch/search/aggregations/support/PathTests.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.search.aggregations.support;
 
 import org.elasticsearch.search.aggregations.AggregationExecutionException;
+import org.elasticsearch.test.ElasticsearchTestCase;
 import org.junit.Test;
 
 import java.util.ArrayList;
@@ -32,7 +33,7 @@ import static org.junit.Assert.fail;
 /**
  *
  */
-public class PathTests {
+public class PathTests extends ElasticsearchTestCase {
 
     @Test
     public void testInvalidPaths() throws Exception {

src/test/java/org/elasticsearch/snapshots/SnapshotUtilsTests.java

@@ -20,6 +20,7 @@ package org.elasticsearch.snapshots;
 
 import com.google.common.collect.ImmutableList;
 import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.test.ElasticsearchTestCase;
 import org.junit.Test;
 
 import static org.hamcrest.MatcherAssert.assertThat;
@@ -27,7 +28,7 @@ import static org.hamcrest.Matchers.containsInAnyOrder;
 
 /**
  */
-public class SnapshotUtilsTests {
+public class SnapshotUtilsTests extends ElasticsearchTestCase {
     @Test
     public void testIndexNameFiltering() {
         assertIndexNameFiltering(new String[]{"foo", "bar", "baz"}, new String[]{}, new String[]{"foo", "bar", "baz"});

src/test/java/org/elasticsearch/test/ElasticsearchBackwardsCompatIntegrationTest.java

@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.test;
 
+import org.apache.lucene.util.AbstractRandomizedTest;
 import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.discovery.DiscoveryModule;
@@ -43,12 +44,12 @@ import java.io.IOException;
  * <p>
  *   Note: this base class is still experimental and might have bugs or leave external processes running behind.
  * </p>
- * Backwards compatibility tests are disabled by default via {@link BackwardsCompatibilityTest} annotation.
+ * Backwards compatibility tests are disabled by default via {@link org.apache.lucene.util.AbstractRandomizedTest.Backwards} annotation.
  * The following system variables control the test execution:
  * <ul>
  *     <li>
  *          <tt>{@value #TESTS_BACKWARDS_COMPATIBILITY}</tt> enables / disables
- *          tests annotated with {@link BackwardsCompatibilityTest} (defaults to
+ *          tests annotated with {@link org.apache.lucene.util.AbstractRandomizedTest.Backwards} (defaults to
  *          <tt>false</tt>)
  *     </li>
  *     <li>
@@ -66,7 +67,7 @@ import java.io.IOException;
  *
  */
 // the transportClientRatio is tricky here since we don't fully control the cluster nodes
-@ElasticsearchBackwardsCompatIntegrationTest.BackwardsCompatibilityTest
+@AbstractRandomizedTest.Backwards
 @ElasticsearchIntegrationTest.ClusterScope(minNumDataNodes = 0, maxNumDataNodes = 2, scope = ElasticsearchIntegrationTest.Scope.SUITE, numClientNodes = 0, transportClientRatio = 0.0)
 @Ignore
 public abstract class ElasticsearchBackwardsCompatIntegrationTest extends ElasticsearchIntegrationTest {

src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java

@@ -168,7 +168,7 @@ import static org.hamcrest.Matchers.equalTo;
  * </p>
  */
 @Ignore
-@AbstractRandomizedTest.IntegrationTests
+@AbstractRandomizedTest.Integration
 public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase {
     private static TestCluster GLOBAL_CLUSTER;
     /**