Upgrade to Lucene-7.4.0-snapshot-59f2b7aec2 (#30726)

This snapshot resolves issues related to ShrinkIndexIT.

buildSrc/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 7.0.0-alpha1
-lucene            = 7.4.0-snapshot-6705632810
+lucene            = 7.4.0-snapshot-59f2b7aec2
 
 # optional dependencies
 spatial4j         = 0.7

modules/lang-expression/licenses/lucene-expressions-7.4.0-snapshot-59f2b7aec2.jar.sha1

@@ -0,0 +1 @@
+a3dba337d06e1f5930cb7ae638c1655b99ce0cb7

modules/lang-expression/licenses/lucene-expressions-7.4.0-snapshot-6705632810.jar.sha1

@@ -1 +0,0 @@
-f72ad4b6474c2d59b0eed0ca84eddd1f99d29129

plugins/analysis-icu/licenses/lucene-analyzers-icu-7.4.0-snapshot-59f2b7aec2.jar.sha1

@@ -0,0 +1 @@
+473a7f4d955f132bb498482648266653f8da85bd

plugins/analysis-icu/licenses/lucene-analyzers-icu-7.4.0-snapshot-6705632810.jar.sha1

@@ -1 +0,0 @@
-b4e19c53f29fa9b40bd7ad12ff598e3f08d507a3

plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.4.0-snapshot-59f2b7aec2.jar.sha1

@@ -0,0 +1 @@
+c5a72b9a790e2552248c8bbb36af47c4c399ba27

plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.4.0-snapshot-6705632810.jar.sha1

@@ -1 +0,0 @@
-23dd8cb3834f3641d9b3e8bc3d38281389a597bc

plugins/analysis-nori/licenses/lucene-analyzers-nori-7.4.0-snapshot-59f2b7aec2.jar.sha1

@@ -0,0 +1 @@
+14f680ab9b886c7c5224ff682a7fa70b6df44a05

plugins/analysis-nori/licenses/lucene-analyzers-nori-7.4.0-snapshot-6705632810.jar.sha1

@@ -1 +0,0 @@
-e8119a17448a6f5512ded0bd2a6faa7fc8e70890

plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.4.0-snapshot-59f2b7aec2.jar.sha1

@@ -0,0 +1 @@
+e033c68c9ec1ba9cd8439758adf7eb5fee22acef

plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.4.0-snapshot-6705632810.jar.sha1

@@ -1 +0,0 @@
-336d9ac698066b8cf8a448f193e4a29ef163baa8

plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.4.0-snapshot-59f2b7aec2.jar.sha1

@@ -0,0 +1 @@
+08df0a5029f11c109b22064dec78c05dfa25f9e3

plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.4.0-snapshot-6705632810.jar.sha1

@@ -1 +0,0 @@
-e1e77951a83fc6a9deab884773314992fefa14f3

plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.4.0-snapshot-59f2b7aec2.jar.sha1

@@ -0,0 +1 @@
+a9d1819b2b13f134f6a605ab5a59ce3c602c0460

plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.4.0-snapshot-6705632810.jar.sha1

@@ -1 +0,0 @@
-d4da149a16673c6326f4898ad877756259f676f8

plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.4.0-snapshot-59f2b7aec2.jar.sha1

@@ -0,0 +1 @@
+47bc91ccb0cdf0c1c404646ffe0d5fd6b020a4ab

plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.4.0-snapshot-6705632810.jar.sha1

@@ -1 +0,0 @@
-ab4141b43cc6c2680d5f5a0b5086299f38ebec4d

server/licenses/lucene-analyzers-common-7.4.0-snapshot-59f2b7aec2.jar.sha1

@@ -0,0 +1 @@
+b70d03784d06a643e096fae4d959200aa246ba16

server/licenses/lucene-analyzers-common-7.4.0-snapshot-6705632810.jar.sha1

@@ -1 +0,0 @@
-f465718b3db829e7660009aac2c1211fd5d74ca0

server/licenses/lucene-backward-codecs-7.4.0-snapshot-59f2b7aec2.jar.sha1

@@ -0,0 +1 @@
+d660a63ac0f7ab2772a45ae518518472bf620620

server/licenses/lucene-backward-codecs-7.4.0-snapshot-6705632810.jar.sha1

@@ -1 +0,0 @@
-d502441e830e1a9d30270442f8e3fd8317fe7bba

server/licenses/lucene-core-7.4.0-snapshot-59f2b7aec2.jar.sha1

@@ -0,0 +1 @@
+bf8f9e8284a54af18545574cb4a530da0deb968a

server/licenses/lucene-core-7.4.0-snapshot-6705632810.jar.sha1

@@ -1 +0,0 @@
-5167fb0a14434cb10ec3224e9e32ca668e9f9ad4

server/licenses/lucene-grouping-7.4.0-snapshot-59f2b7aec2.jar.sha1

@@ -0,0 +1 @@
+9eaae9dcd4ec88227475cb81d3be9afa767f1b22

server/licenses/lucene-grouping-7.4.0-snapshot-6705632810.jar.sha1

@@ -1 +0,0 @@
-488aeecf49413b63a404989ae00b07b20951e76e

server/licenses/lucene-highlighter-7.4.0-snapshot-59f2b7aec2.jar.sha1

@@ -0,0 +1 @@
+cd15f0008742c84899d678cb0cecda06d0a6d63e

server/licenses/lucene-highlighter-7.4.0-snapshot-6705632810.jar.sha1

@@ -1 +0,0 @@
-107755edd67cddb3fb9817de50c0bed3a10da19c

server/licenses/lucene-join-7.4.0-snapshot-59f2b7aec2.jar.sha1

@@ -0,0 +1 @@
+5ce38b8610a7f402f2da3b0e408e508151d979c5

server/licenses/lucene-join-7.4.0-snapshot-6705632810.jar.sha1

@@ -1 +0,0 @@
-9226fab3b9c6250af52b87061f637c0f8e3114b6

server/licenses/lucene-memory-7.4.0-snapshot-59f2b7aec2.jar.sha1

@@ -0,0 +1 @@
+53819f03a07050a4af28361d64395c86f2cea008

server/licenses/lucene-memory-7.4.0-snapshot-6705632810.jar.sha1

@@ -1 +0,0 @@
-2b7bf384c1933225972f04224d867ec800f5e3a7

server/licenses/lucene-misc-7.4.0-snapshot-59f2b7aec2.jar.sha1

@@ -0,0 +1 @@
+8cdc0e2b65d146ed11f4d2507109e530d59ff33d

server/licenses/lucene-misc-7.4.0-snapshot-6705632810.jar.sha1

@@ -1 +0,0 @@
-18b770c35db8757dc036b1506870a4ddaad7b1ab

server/licenses/lucene-queries-7.4.0-snapshot-59f2b7aec2.jar.sha1

@@ -0,0 +1 @@
+e56090463703112ad64ad457d18bae9a5b2966b8

server/licenses/lucene-queries-7.4.0-snapshot-6705632810.jar.sha1

@@ -1 +0,0 @@
-683f6436938c67709d0c665c9e1fdef7bd893e4a

server/licenses/lucene-queryparser-7.4.0-snapshot-59f2b7aec2.jar.sha1

@@ -0,0 +1 @@
+9faf974b77058e44a6d35e956db4f5fb67389dfa

server/licenses/lucene-queryparser-7.4.0-snapshot-6705632810.jar.sha1

@@ -1 +0,0 @@
-1df20ba64b9aa68f1fa9a15c9ff75f87f94dec47

server/licenses/lucene-sandbox-7.4.0-snapshot-59f2b7aec2.jar.sha1

@@ -0,0 +1 @@
+b852b1fe70ef70736b2b1a9ad57eb93cbaed0423

server/licenses/lucene-sandbox-7.4.0-snapshot-6705632810.jar.sha1

@@ -1 +0,0 @@
-895ca714fc62b66ba63d43931730cdc4ef56d35f

server/licenses/lucene-spatial-7.4.0-snapshot-59f2b7aec2.jar.sha1

@@ -0,0 +1 @@
+d2fa99ec7140fcf35db16ac1feb78ef142750d39

server/licenses/lucene-spatial-7.4.0-snapshot-6705632810.jar.sha1

@@ -1 +0,0 @@
-95ab7e9421bbeb8229d83ac72700b37a521fdf4f

server/licenses/lucene-spatial-extras-7.4.0-snapshot-59f2b7aec2.jar.sha1

@@ -0,0 +1 @@
+c9963f60d3a0924b877a6f910650c5f2384822a0

server/licenses/lucene-spatial-extras-7.4.0-snapshot-6705632810.jar.sha1

@@ -1 +0,0 @@
-773ff8c8425d32609ccec6956759ad377dfb8f6b

server/licenses/lucene-spatial3d-7.4.0-snapshot-59f2b7aec2.jar.sha1

@@ -0,0 +1 @@
+3f33ba54da5e0e125f4c5ef7dd800dd6185e4f61

server/licenses/lucene-spatial3d-7.4.0-snapshot-6705632810.jar.sha1

@@ -1 +0,0 @@
-ea711541e243ee768f950041e6e2843d0cc5e695

server/licenses/lucene-suggest-7.4.0-snapshot-59f2b7aec2.jar.sha1

@@ -0,0 +1 @@
+bb3c18c987395dae6fe63744f5a50fd367ea5a74

server/licenses/lucene-suggest-7.4.0-snapshot-6705632810.jar.sha1

@@ -1 +0,0 @@
-2ca005cf25722ba3777ed93f720f40c937081fa6

server/src/main/java/org/elasticsearch/index/shard/ElasticsearchMergePolicy.java

@@ -20,9 +20,9 @@
 package org.elasticsearch.index.shard;
 
 import org.apache.logging.log4j.Logger;
+import org.apache.lucene.index.FilterMergePolicy;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.MergePolicy;
-import org.apache.lucene.index.MergePolicyWrapper;
 import org.apache.lucene.index.SegmentCommitInfo;
 import org.apache.lucene.index.SegmentInfos;
 import org.elasticsearch.Version;
@@ -44,7 +44,7 @@ import java.util.Map;
  * For now, this {@link MergePolicy} takes care of moving versions that used to
  * be stored as payloads to numeric doc values.
  */
-public final class ElasticsearchMergePolicy extends MergePolicyWrapper {
+public final class ElasticsearchMergePolicy extends FilterMergePolicy {
 
     private static Logger logger = Loggers.getLogger(ElasticsearchMergePolicy.class);
 

server/src/main/java/org/elasticsearch/index/store/Store.java

@@ -759,17 +759,6 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
         public String toString() {
             return "store(" + in.toString() + ")";
         }
-
-        @Override
-        public boolean checkPendingDeletions() throws IOException {
-            if (super.checkPendingDeletions()) {
-                deletesLogger.warn("directory has still pending deletes");
-            }
-            // we skip this check since our IW usage always goes forward.
-            // we still might run into situations where we have pending deletes ie. in shrink / split case
-            // and that will cause issues on windows since we open multiple IW instance one after another during the split/shrink recovery
-            return false;
-        }
     }
 
     /**

server/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java

@@ -23,7 +23,6 @@ import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.search.SortedSetSelector;
 import org.apache.lucene.search.SortedSetSortField;
-import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteResponse;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
@@ -84,7 +83,6 @@ public class ShrinkIndexIT extends ESIntegTestCase {
         return Arrays.asList(InternalSettingsPlugin.class);
     }
 
-    @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/LUCENE-8318")
     public void testCreateShrinkIndexToN() {
         int[][] possibleShardSplits = new int[][] {{8,4,2}, {9, 3, 1}, {4, 2, 1}, {15,5,1}};
         int[] shardSplits = randomFrom(possibleShardSplits);

test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java

@@ -217,6 +217,9 @@ public abstract class AnalysisFactoryTestCase extends ESTestCase {
         // should we expose it, or maybe think about higher level integration of the
         // fake term frequency feature (LUCENE-7854)
         .put("delimitedtermfrequency",    Void.class)
+        // LUCENE-8273: ConditionalTokenFilter allows analysis chains to skip
+        // particular token filters based on the attributes of the current token.
+        .put("termexclusion", Void.class)
 
         .immutableMap();
 

x-pack/plugin/sql/jdbc/licenses/lucene-core-7.4.0-snapshot-59f2b7aec2.jar.sha1

@@ -0,0 +1 @@
+bf8f9e8284a54af18545574cb4a530da0deb968a

x-pack/plugin/sql/jdbc/licenses/lucene-core-7.4.0-snapshot-6705632810.jar.sha1

@@ -1 +0,0 @@
-5167fb0a14434cb10ec3224e9e32ca668e9f9ad4

x-pack/plugin/sql/sql-proto/licenses/lucene-core-7.4.0-snapshot-59f2b7aec2.jar.sha1

@@ -0,0 +1 @@
+bf8f9e8284a54af18545574cb4a530da0deb968a

x-pack/plugin/sql/sql-proto/licenses/lucene-core-7.4.0-snapshot-6705632810.jar.sha1

@@ -1 +0,0 @@
-5167fb0a14434cb10ec3224e9e32ca668e9f9ad4