Upgrade to lucene-6.4.0-snapshot-ec38570 (#21853)

Set lucene version to 6.4.0-snapshot-ec38570 and update all the sha1s/license
Fix invalid combo after upgrade in query_string query. split_on_whitespace=false is disallowed if auto_generate_phrase_queries=true
Adapt the expectations of some tests to the new format of the Lucene explain output

buildSrc/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 6.0.0-alpha1
-lucene            = 6.3.0
+lucene            = 6.4.0-snapshot-ec38570
 
 # optional dependencies
 spatial4j         = 0.6

core/licenses/lucene-analyzers-common-6.3.0.jar.sha1

@@ -1 +0,0 @@
-494aed699af238c3872a6b65e17939e9cb7ddbe0

core/licenses/lucene-analyzers-common-6.4.0-snapshot-ec38570.jar.sha1

@@ -0,0 +1 @@
+770114e0188dd8b4f30e5878b4f6c8677cecf1be

core/licenses/lucene-backward-codecs-6.3.0.jar.sha1

@@ -1 +0,0 @@
-77dede7dff1b833ca2e92d8ab137edb209354d9b

core/licenses/lucene-backward-codecs-6.4.0-snapshot-ec38570.jar.sha1

@@ -0,0 +1 @@
+f4eb0257e8419beaa9f84da6a51375fda4e491f2

core/licenses/lucene-core-6.3.0.jar.sha1

@@ -1 +0,0 @@
-d3c87ea89e2f83e401f9cc7f14e4c43945f7f1e1

core/licenses/lucene-core-6.4.0-snapshot-ec38570.jar.sha1

@@ -0,0 +1 @@
+c80ad16cd36c41012abb8a8bb1c7328c6d680b4a

core/licenses/lucene-grouping-6.3.0.jar.sha1

@@ -1 +0,0 @@
-2c96d59e318ea66838aeb9c5cfb8b4d27b40953c

core/licenses/lucene-grouping-6.4.0-snapshot-ec38570.jar.sha1

@@ -0,0 +1 @@
+070d4e370f4fe0b8a04b2bce5b4381201b0c783f

core/licenses/lucene-highlighter-6.3.0.jar.sha1

@@ -1 +0,0 @@
-4f154d8badfe47fe45503c18fb30f2177f758794

core/licenses/lucene-highlighter-6.4.0-snapshot-ec38570.jar.sha1

@@ -0,0 +1 @@
+131d9a86f5943675493a85def0e692842f396458

core/licenses/lucene-join-6.3.0.jar.sha1

@@ -1 +0,0 @@
-79b898117dcfde2981ec6806e420ff218842eca8

core/licenses/lucene-join-6.4.0-snapshot-ec38570.jar.sha1

@@ -0,0 +1 @@
+385b2202036b50a764e4d2b032e21496b74a1c8e

core/licenses/lucene-memory-6.3.0.jar.sha1

@@ -1 +0,0 @@
-89edeb404e507d640cb13903acff6953199704a2

core/licenses/lucene-memory-6.4.0-snapshot-ec38570.jar.sha1

@@ -0,0 +1 @@
+e8742a44ef4849a17d5e59ef36e9a52a8f2370c2

core/licenses/lucene-misc-6.3.0.jar.sha1

@@ -1 +0,0 @@
-02d0e1f5a9df15ac911ad495bad5ea253ab50a9f

core/licenses/lucene-misc-6.4.0-snapshot-ec38570.jar.sha1

@@ -0,0 +1 @@
+7ce2e4948fb66393a34f4200a6131cfde43e47bd

core/licenses/lucene-queries-6.3.0.jar.sha1

@@ -1 +0,0 @@
-eb7938233c8103223069c7b5b5f785b4d20ddafa

core/licenses/lucene-queries-6.4.0-snapshot-ec38570.jar.sha1

@@ -0,0 +1 @@
+6c1c385a597ce797b0049d9b2281b09593e1488a

core/licenses/lucene-queryparser-6.3.0.jar.sha1

@@ -1 +0,0 @@
-e979fb02155cbe81a8d335d6dc41d2ef06be68b6

core/licenses/lucene-queryparser-6.4.0-snapshot-ec38570.jar.sha1

@@ -0,0 +1 @@
+fafaa22906c067e6894f9f2b18ad03ded98e2f38

core/licenses/lucene-sandbox-6.3.0.jar.sha1

@@ -1 +0,0 @@
-257387c45c6fa2b77fd6931751f93fdcd798ced4

core/licenses/lucene-sandbox-6.4.0-snapshot-ec38570.jar.sha1

@@ -0,0 +1 @@
+19c64a84617f42bb4c11b1e266df4009cd37fdd0

core/licenses/lucene-spatial-6.3.0.jar.sha1

@@ -1 +0,0 @@
-3cf5fe5402b5e34b240b73501c9e97a82428259e

core/licenses/lucene-spatial-6.4.0-snapshot-ec38570.jar.sha1

@@ -0,0 +1 @@
+bc8613fb61c0ae95dd3680b0f65e3380c3fd0d6c

core/licenses/lucene-spatial-extras-6.3.0.jar.sha1

@@ -1 +0,0 @@
-1b77ef3740dc885c62d5966fbe9aea1199d344fb

core/licenses/lucene-spatial-extras-6.4.0-snapshot-ec38570.jar.sha1

@@ -0,0 +1 @@
+0fa2c3e722294e863f3c70a15e97a18397391fb4

core/licenses/lucene-spatial3d-6.3.0.jar.sha1

@@ -1 +0,0 @@
-aa94b4a8636b3633008640cc5155ad354aebcea5

core/licenses/lucene-spatial3d-6.4.0-snapshot-ec38570.jar.sha1

@@ -0,0 +1 @@
+db74c6313965ffdd10d9b19be2eed4ae2c76d2e3

core/licenses/lucene-suggest-6.3.0.jar.sha1

@@ -1 +0,0 @@
-ed5d8ee5cd7edcad5d4ffca2b4540ccc844e9bb0

core/licenses/lucene-suggest-6.4.0-snapshot-ec38570.jar.sha1

@@ -0,0 +1 @@
+b85ae1121b5fd56df985615a3cdd7b3879e9b92d

core/src/main/java/org/elasticsearch/Version.java

@@ -103,7 +103,7 @@ public class Version {
     public static final Version V_5_2_0_UNRELEASED = new Version(V_5_2_0_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_6_3_0);
     public static final int V_6_0_0_alpha1_ID_UNRELEASED = 6000001;
     public static final Version V_6_0_0_alpha1_UNRELEASED =
-        new Version(V_6_0_0_alpha1_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_6_3_0);
+        new Version(V_6_0_0_alpha1_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_6_4_0);
     public static final Version CURRENT = V_6_0_0_alpha1_UNRELEASED;
 
     // unreleased versions must be added to the above list with the suffix _UNRELEASED (with the exception of CURRENT)

core/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java

@@ -937,6 +937,10 @@ public class QueryStringQueryBuilder extends AbstractQueryBuilder<QueryStringQue
     protected Query doToQuery(QueryShardContext context) throws IOException {
         //TODO would be nice to have all the settings in one place: some change though at query execution time
         //e.g. field names get expanded to concrete names, defaults get resolved sometimes to settings values etc.
+        if (splitOnWhitespace == false && autoGeneratePhraseQueries) {
+            throw new IllegalArgumentException("it is disallowed to disable [split_on_whitespace] " +
+                "if [auto_generate_phrase_queries] is activated");
+        }
         QueryParserSettings qpSettings;
         if (this.escape) {
             qpSettings = new QueryParserSettings(org.apache.lucene.queryparser.classic.QueryParser.escape(this.queryString));

core/src/main/resources/org/elasticsearch/bootstrap/security.policy

@@ -31,7 +31,7 @@ grant codeBase "${codebase.securesm-1.1.jar}" {
 //// Very special jar permissions:
 //// These are dangerous permissions that we don't want to grant to everything.
 
-grant codeBase "${codebase.lucene-core-6.3.0.jar}" {
+grant codeBase "${codebase.lucene-core-6.4.0-snapshot-ec38570.jar}" {
   // needed to allow MMapDirectory's "unmap hack" (die unmap hack, die)
   // java 8 package
   permission java.lang.RuntimePermission "accessClassInPackage.sun.misc";
@@ -42,7 +42,7 @@ grant codeBase "${codebase.lucene-core-6.3.0.jar}" {
   permission java.lang.RuntimePermission "accessDeclaredMembers";
 };
 
-grant codeBase "${codebase.lucene-misc-6.3.0.jar}" {
+grant codeBase "${codebase.lucene-misc-6.4.0-snapshot-ec38570.jar}" {
   // needed to allow shard shrinking to use hard-links if possible via lucenes HardlinkCopyDirectoryWrapper
   permission java.nio.file.LinkPermission "hard";
 };
@@ -92,10 +92,10 @@ grant {
   permission java.lang.RuntimePermission "modifyThreadGroup";
 
   // needed by ExceptionSerializationTests and RestTestCase for
-  // some hackish things they do. otherwise only needed by groovy 
+  // some hackish things they do. otherwise only needed by groovy
   // (TODO: clean this up?)
   permission java.lang.RuntimePermission "getProtectionDomain";
-  
+
   // needed by HotThreads and potentially more
   // otherwise can be provided only to test libraries
   permission java.lang.RuntimePermission "getStackTrace";

core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy

@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
- 
+
 //// additional test framework permissions.
 //// These are mock objects and test management that we allow test framework libs
 //// to provide on our behalf. But tests themselves cannot do this stuff!
@@ -33,7 +33,7 @@ grant codeBase "${codebase.securemock-1.2.jar}" {
   permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
 };
 
-grant codeBase "${codebase.lucene-test-framework-6.3.0.jar}" {
+grant codeBase "${codebase.lucene-test-framework-6.4.0-snapshot-ec38570.jar}" {
   // needed by RamUsageTester
   permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
   // needed for testing hardlinks in StoreRecoveryTests since we install MockFS

core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java

@@ -150,7 +150,10 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
         if (randomBoolean()) {
             queryStringQueryBuilder.timeZone(randomDateTimeZone().getID());
         }
-        queryStringQueryBuilder.splitOnWhitespace(randomBoolean());
+        if (queryStringQueryBuilder.autoGeneratePhraseQueries() == false) {
+            // setSplitOnWhitespace(false) is disallowed when getAutoGeneratePhraseQueries() == true
+            queryStringQueryBuilder.splitOnWhitespace(randomBoolean());
+        }
         return queryStringQueryBuilder;
     }
 
@@ -764,4 +767,14 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
         assertThat(e.getMessage(),
                 containsString("cannot use [all_fields] parameter in conjunction with [default_field] or [fields]"));
     }
+
+    public void testInvalidCombo() throws IOException {
+        QueryStringQueryBuilder builder = new QueryStringQueryBuilder("foo bar");
+        builder.autoGeneratePhraseQueries(true);
+        builder.splitOnWhitespace(false);
+        IllegalArgumentException exc =
+            expectThrows(IllegalArgumentException.class, () -> builder.toQuery(createShardContext()));
+        assertEquals(exc.getMessage(),
+            "it is disallowed to disable [split_on_whitespace] if [auto_generate_phrase_queries] is activated");
+    }
 }

core/src/test/java/org/elasticsearch/search/basic/TransportTwoNodesSearchIT.java

@@ -61,7 +61,6 @@ import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
-import static org.hamcrest.Matchers.endsWith;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
@@ -151,8 +150,15 @@ public class TransportTwoNodesSearchIT extends ESIntegTestCase {
                 assertThat(hit.explanation(), notNullValue());
                 assertThat(hit.explanation().getDetails().length, equalTo(1));
                 assertThat(hit.explanation().getDetails()[0].getDetails().length, equalTo(2));
-                assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDescription(),
-                    endsWith("idf(docFreq=100, docCount=100)"));
+                assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails().length, equalTo(2));
+                assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails()[0].getDescription(),
+                    equalTo("docFreq"));
+                assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails()[0].getValue(),
+                    equalTo(100.0f));
+                assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails()[1].getDescription(),
+                    equalTo("docCount"));
+                assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails()[1].getValue(),
+                    equalTo(100.0f));
                 assertThat("id[" + hit.id() + "] -> " + hit.explanation().toString(), hit.id(), equalTo(Integer.toString(100 - total - i - 1)));
             }
             total += hits.length;
@@ -179,8 +185,15 @@ public class TransportTwoNodesSearchIT extends ESIntegTestCase {
                 assertThat(hit.explanation(), notNullValue());
                 assertThat(hit.explanation().getDetails().length, equalTo(1));
                 assertThat(hit.explanation().getDetails()[0].getDetails().length, equalTo(2));
-                assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDescription(),
-                    endsWith("idf(docFreq=100, docCount=100)"));
+                assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails().length, equalTo(2));
+                assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails()[0].getDescription(),
+                    equalTo("docFreq"));
+                assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails()[0].getValue(),
+                    equalTo(100.0f));
+                assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails()[1].getDescription(),
+                    equalTo("docCount"));
+                assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails()[1].getValue(),
+                    equalTo(100.0f));
                 assertThat("id[" + hit.id() + "]", hit.id(), equalTo(Integer.toString(total + i)));
             }
             total += hits.length;
@@ -329,8 +342,15 @@ public class TransportTwoNodesSearchIT extends ESIntegTestCase {
             assertThat(hit.explanation(), notNullValue());
             assertThat(hit.explanation().getDetails().length, equalTo(1));
             assertThat(hit.explanation().getDetails()[0].getDetails().length, equalTo(2));
-            assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDescription(),
-                endsWith("idf(docFreq=100, docCount=100)"));
+            assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails().length, equalTo(2));
+            assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails()[0].getDescription(),
+                equalTo("docFreq"));
+            assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails()[0].getValue(),
+                equalTo(100.0f));
+            assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails()[1].getDescription(),
+                equalTo("docCount"));
+            assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails()[1].getValue(),
+                equalTo(100.0f));
 //            assertThat("id[" + hit.id() + "]", hit.id(), equalTo(Integer.toString(100 - i - 1)));
             assertThat("make sure we don't have duplicates", expectedIds.remove(hit.id()), notNullValue());
         }

docs/reference/query-dsl/query-string-query.asciidoc

@@ -85,8 +85,8 @@ for exact matching. Look <<mixing-exact-search-with-stemming,here>> for a
 comprehensive example.
 
 |`split_on_whitespace` |Whether query text should be split on whitespace prior to analysis.
-                        Instead  the queryparser would parse around only real 'operators'.
-                        Default to `false`.
+Instead  the queryparser would parse around only real 'operators'. Default to `false`.
+It is not allowed to set this option to `false` if `autoGeneratePhraseQueries` is already set to `true`.
 
 |`all_fields` | Perform the query on all fields detected in the mapping that can
 be queried. Will be used by default when the `_all` field is disabled and no

docs/reference/search/explain.asciidoc

@@ -30,52 +30,102 @@ This will yield the following result:
 [source,js]
 --------------------------------------------------
 {
-  "_index" : "twitter",
-  "_type" : "tweet",
-  "_id" : "0",
-  "matched" : true,
-  "explanation" : {
-    "value" : 1.55077,
-    "description" : "sum of:",
-    "details" : [ {
-      "value" : 1.55077,
-      "description" : "weight(message:elasticsearch in 0) [PerFieldSimilarity], result of:",
-      "details" : [ {
-        "value" : 1.55077,
-        "description" : "score(doc=0,freq=1.0 = termFreq=1.0\n), product of:",
-        "details" : [ {
-          "value" : 1.3862944,
-          "description" : "idf(docFreq=1, docCount=5)",
-          "details" : [ ]
-        }, {
-          "value" : 1.1186441,
-          "description" : "tfNorm, computed from:",
-          "details" : [
-            { "value" : 1.0, "description" : "termFreq=1.0", "details" : [ ] },
-            { "value" : 1.2, "description" : "parameter k1", "details" : [ ] },
-            { "value" : 0.75, "description" : "parameter b", "details" : [ ] },
-            { "value" : 5.4, "description" : "avgFieldLength", "details" : [ ] },
-            { "value" : 4.0, "description" : "fieldLength", "details" : [ ] }
-          ]
-        } ]
-      } ]
-    }, {
-      "value" : 0.0,
-      "description" : "match on required clause, product of:",
-      "details" : [ {
-        "value" : 0.0,
-        "description" : "# clause",
-        "details" : [ ]
-      }, {
-        "value" : 1.0,
-        "description" : "*:*, product of:",
-        "details" : [
-          { "value" : 1.0, "description" : "boost", "details" : [ ] },
-          { "value" : 1.0, "description" : "queryNorm", "details" : [ ] }
-        ]
-      } ]
-    } ]
-  }
+   "_index": "twitter",
+   "_type": "tweet",
+   "_id": "0",
+   "matched": true,
+   "explanation": {
+      "value": 1.55077,
+      "description": "sum of:",
+      "details": [
+         {
+            "value": 1.55077,
+            "description": "weight(message:elasticsearch in 0) [PerFieldSimilarity], result of:",
+            "details": [
+               {
+                  "value": 1.55077,
+                  "description": "score(doc=0,freq=1.0 = termFreq=1.0\n), product of:",
+                  "details": [
+                     {
+                        "value": 1.3862944,
+                        "description": "idf, computed as log(1 + (docCount - docFreq + 0.5) / (docFreq + 0.5)) from:",
+                        "details": [
+                           {
+                              "value": 1.0,
+                              "description": "docFreq",
+                              "details": []
+                           },
+                           {
+                              "value": 5.0,
+                              "description": "docCount",
+                              "details": []
+                           }
+                        ]
+                     },
+                     {
+                        "value": 1.1186441,
+                        "description": "tfNorm, computed as (freq * (k1 + 1)) / (freq + k1 * (1 - b + b * fieldLength / avgFieldLength)) from:",
+                        "details": [
+                           {
+                              "value": 1.0,
+                              "description": "termFreq=1.0",
+                              "details": []
+                           },
+                           {
+                              "value": 1.2,
+                              "description": "parameter k1",
+                              "details": []
+                           },
+                           {
+                              "value": 0.75,
+                              "description": "parameter b",
+                              "details": []
+                           },
+                           {
+                              "value": 5.4,
+                              "description": "avgFieldLength",
+                              "details": []
+                           },
+                           {
+                              "value": 4.0,
+                              "description": "fieldLength",
+                              "details": []
+                           }
+                        ]
+                     }
+                  ]
+               }
+            ]
+         },
+         {
+            "value": 0.0,
+            "description": "match on required clause, product of:",
+            "details": [
+               {
+                  "value": 0.0,
+                  "description": "# clause",
+                  "details": []
+               },
+               {
+                  "value": 1.0,
+                  "description": "*:*, product of:",
+                  "details": [
+                     {
+                        "value": 1.0,
+                        "description": "boost",
+                        "details": []
+                     },
+                     {
+                        "value": 1.0,
+                        "description": "queryNorm",
+                        "details": []
+                     }
+                  ]
+               }
+            ]
+         }
+      ]
+   }
 }
 --------------------------------------------------
 // TESTRESPONSE

modules/lang-expression/licenses/lucene-expressions-6.3.0.jar.sha1

@@ -1 +0,0 @@
-f9847cdbdd355f9f96993c4c322d6b453f4e84a8

modules/lang-expression/licenses/lucene-expressions-6.4.0-snapshot-ec38570.jar.sha1

@@ -0,0 +1 @@
+031d34e0a604a7cbb5c8ba816d49d9f622adaa3f

plugins/analysis-icu/licenses/lucene-analyzers-icu-6.3.0.jar.sha1

@@ -1 +0,0 @@
-467d808656db028faa3cbc86d386dbf6164a835c

plugins/analysis-icu/licenses/lucene-analyzers-icu-6.4.0-snapshot-ec38570.jar.sha1

@@ -0,0 +1 @@
+0850319baf063c5ee54aecabeaddb95efde8711b

plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.3.0.jar.sha1

@@ -1 +0,0 @@
-bea02277bff7fa0f4d93e6abca94eaf0eec9c84f

plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.4.0-snapshot-ec38570.jar.sha1

@@ -0,0 +1 @@
+3a2af1d2e80b9901b3e950f5ac1b6cd1eb408fd3

plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.3.0.jar.sha1

@@ -1 +0,0 @@
-657a1409f539b4a20b5487496a8e4471b33902fd

plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.4.0-snapshot-ec38570.jar.sha1

@@ -0,0 +1 @@
+7e9243da1482f88a91bd5239316b571259d24341

plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.3.0.jar.sha1

@@ -1 +0,0 @@
-47792194b04e8cd61c3667da50a38adae257b19a

plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.4.0-snapshot-ec38570.jar.sha1

@@ -0,0 +1 @@
+2ead714733bb3cc90e9792d76021497946d5af09

plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.3.0.jar.sha1

@@ -1 +0,0 @@
-bcf535520b92821cf04486031214d35d7405571c

plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.4.0-snapshot-ec38570.jar.sha1

@@ -0,0 +1 @@
+9a8f3b58e6c672276331f54b5c3be49c8014ec5c

plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-6.3.0.jar.sha1

@@ -1 +0,0 @@
-82ed82174fae75f93741b8418046bc94e50434f8

plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-6.4.0-snapshot-ec38570.jar.sha1

@@ -0,0 +1 @@
+763b3144b9bc53328e923242a3c6614903ee2d7e