Upgrade to Lucene-7.5.0-snapshot-608f0277b0 (#32390)

The main highlight is the removal of the reclaim_deletes_weight in the TieredMergePolicy.
The es setting index.merge.policy.reclaim_deletes_weight is deprecated in this commit and the value is ignored. The new merge policy setting setDeletesPctAllowed should be added in a follow up.

buildSrc/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 7.0.0-alpha1
-lucene            = 7.4.0
+lucene            = 7.5.0-snapshot-608f0277b0
 
 # optional dependencies
 spatial4j         = 0.7

docs/Versions.asciidoc

@@ -1,7 +1,7 @@
 :version:               7.0.0-alpha1
 :major-version:         7.x
-:lucene_version:        7.4.0
-:lucene_version_path:   7_4_0
+:lucene_version:        7.5.0
+:lucene_version_path:   7_5_0
 :branch:                master
 :jdk:                   1.8.0_131
 :jdk_major:             8

modules/lang-expression/licenses/lucene-expressions-7.4.0.jar.sha1

@@ -1 +0,0 @@
-9f0a326f7ec1671ffb07f95b27f1a5812b7dc1c3

modules/lang-expression/licenses/lucene-expressions-7.5.0-snapshot-608f0277b0.jar.sha1

@@ -0,0 +1 @@
+bd7d8078a2d0ad11a24f54156cc015630c96858a

plugins/analysis-icu/build.gradle

@@ -30,7 +30,7 @@ forbiddenApis {
 
 dependencies {
   compile "org.apache.lucene:lucene-analyzers-icu:${versions.lucene}"
-  compile 'com.ibm.icu:icu4j:61.1'
+  compile 'com.ibm.icu:icu4j:62.1'
 }
 
 dependencyLicenses {

plugins/analysis-icu/licenses/icu4j-61.1.jar.sha1

@@ -1 +0,0 @@
-28d33b5e44e72edcc66a5da7a34a42147f38d987

plugins/analysis-icu/licenses/icu4j-62.1.jar.sha1

@@ -0,0 +1 @@
+7a4d00d5ec5febd252a6182e8b6e87a0a9821f81

plugins/analysis-icu/licenses/lucene-analyzers-icu-7.4.0.jar.sha1

@@ -1 +0,0 @@
-394e811e9d9bf0b9fba837f7ceca9e8f3e39d1c2

plugins/analysis-icu/licenses/lucene-analyzers-icu-7.5.0-snapshot-608f0277b0.jar.sha1

@@ -0,0 +1 @@
+7a37816def72a748416c4ae8b0f6817e30efb99f

plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.4.0.jar.sha1

@@ -1 +0,0 @@
-5cd56acfa16ba20e19b5d21d90b510eada841431

plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.5.0-snapshot-608f0277b0.jar.sha1

@@ -0,0 +1 @@
+ca7437178cdbf7b8bfe0d75c75e3c8eb93925724

plugins/analysis-nori/licenses/lucene-analyzers-nori-7.4.0.jar.sha1

@@ -1 +0,0 @@
-db7b56f4cf533ad9022d2312c5ee48331edccca3

plugins/analysis-nori/licenses/lucene-analyzers-nori-7.5.0-snapshot-608f0277b0.jar.sha1

@@ -0,0 +1 @@
+3f5dec44f380d6d58bc1c8aec51964fcb5390b60

plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.4.0.jar.sha1

@@ -1 +0,0 @@
-e8dba4d28a595eab2e8fb6095d1ac5f2d3872144

plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.5.0-snapshot-608f0277b0.jar.sha1

@@ -0,0 +1 @@
+453bf1d60df0415439095624e0b3e42492ad4716

plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.4.0.jar.sha1

@@ -1 +0,0 @@
-1243c771ee824c46a3d66ae3e4256d919fc06fbe

plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.5.0-snapshot-608f0277b0.jar.sha1

@@ -0,0 +1 @@
+70095a45257bca9f46629b5fb6cedf9eff5e2b07

plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.4.0.jar.sha1

@@ -1 +0,0 @@
-c783794b0d20d8dc1285edc7701f386b1f0e2fb8

plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.5.0-snapshot-608f0277b0.jar.sha1

@@ -0,0 +1 @@
+7199d6962d268b7877f7b5160e98e4ff21cce5c7

plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.4.0.jar.sha1

@@ -1 +0,0 @@
-9438efa504a89afb6cb4c66448c257f865164d23

plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.5.0-snapshot-608f0277b0.jar.sha1

@@ -0,0 +1 @@
+12aff508d39d206a1aead5013ecd11882062eb06

plugins/store-smb/src/test/java/org/elasticsearch/index/store/SmbMMapDirectoryTests.java

@@ -30,4 +30,12 @@ public class SmbMMapDirectoryTests extends EsBaseDirectoryTestCase {
     protected Directory getDirectory(Path file) throws IOException {
         return new SmbDirectoryWrapper(new MMapDirectory(file));
     }
+
+    @Override
+    public void testCreateOutputForExistingFile() throws IOException {
+        /**
+         * This test is disabled because {@link SmbDirectoryWrapper} opens existing file
+         * with an explicit StandardOpenOption.TRUNCATE_EXISTING option.
+         */
+    }
 }

plugins/store-smb/src/test/java/org/elasticsearch/index/store/SmbSimpleFSDirectoryTests.java

@@ -30,4 +30,12 @@ public class SmbSimpleFSDirectoryTests extends EsBaseDirectoryTestCase {
     protected Directory getDirectory(Path file) throws IOException {
         return new SmbDirectoryWrapper(new SimpleFSDirectory(file));
     }
+
+    @Override
+    public void testCreateOutputForExistingFile() throws IOException {
+        /**
+         * This test is disabled because {@link SmbDirectoryWrapper} opens existing file
+         * with an explicit StandardOpenOption.TRUNCATE_EXISTING option.
+         */
+    }
 }

server/licenses/lucene-analyzers-common-7.4.0.jar.sha1

@@ -1 +0,0 @@
-e1afb580df500626a1c695e0fc9a7e8a8f58bcac

server/licenses/lucene-analyzers-common-7.5.0-snapshot-608f0277b0.jar.sha1

@@ -0,0 +1 @@
+d27958843ca118db2ffd2c242ae3761bd5a47328

server/licenses/lucene-backward-codecs-7.4.0.jar.sha1

@@ -1 +0,0 @@
-a6ad941ef1fdad48673ed511631b7e48a9456bf7

server/licenses/lucene-backward-codecs-7.5.0-snapshot-608f0277b0.jar.sha1

@@ -0,0 +1 @@
+7ea220ba8e4accb8b04e280463042ad470e23bc0

server/licenses/lucene-core-7.4.0.jar.sha1

@@ -1 +0,0 @@
-730d9ac80436c8cbc0b2a8a749259be536b97316

server/licenses/lucene-core-7.5.0-snapshot-608f0277b0.jar.sha1

@@ -0,0 +1 @@
+471096d6e92338b208aa91f3a85feb2f9cfc4afd

server/licenses/lucene-grouping-7.4.0.jar.sha1

@@ -1 +0,0 @@
-56f99858a4421a517b52da36a222debcccab80c6

server/licenses/lucene-grouping-7.5.0-snapshot-608f0277b0.jar.sha1

@@ -0,0 +1 @@
+f0af947c60d24f779c22f774e81ebd7dd91cc932

server/licenses/lucene-highlighter-7.4.0.jar.sha1

@@ -1 +0,0 @@
-5266b45d7f049662817d739881765904621876d0

server/licenses/lucene-highlighter-7.5.0-snapshot-608f0277b0.jar.sha1

@@ -0,0 +1 @@
+fbc83ac5a0139ed7e7faf6c95a2718f46f28c641

server/licenses/lucene-join-7.4.0.jar.sha1

@@ -1 +0,0 @@
-c77154d18c4944ceb6ce0741060632f57d623fdc

server/licenses/lucene-join-7.5.0-snapshot-608f0277b0.jar.sha1

@@ -0,0 +1 @@
+30adfe493982b0db059dc243e269eea38d850d46

server/licenses/lucene-memory-7.4.0.jar.sha1

@@ -1 +0,0 @@
-186ff981feec1bdbf1a6236e786ec171b5fbe3e0

server/licenses/lucene-memory-7.5.0-snapshot-608f0277b0.jar.sha1

@@ -0,0 +1 @@
+656f304261d9aad05070fb68593beffafe9147e3

server/licenses/lucene-misc-7.4.0.jar.sha1

@@ -1 +0,0 @@
-bf844bb6f6d84da19e8c79ce5fbb4cf6d00f2611

server/licenses/lucene-misc-7.5.0-snapshot-608f0277b0.jar.sha1

@@ -0,0 +1 @@
+8bf22ad81a7480c255b55bada401eb131bfdb4df

server/licenses/lucene-queries-7.4.0.jar.sha1

@@ -1 +0,0 @@
-229a50e6d9d4db076f671c230d493000c6e2972c

server/licenses/lucene-queries-7.5.0-snapshot-608f0277b0.jar.sha1

@@ -0,0 +1 @@
+edb3de4d68a34c1e1ca08f79fe4d103b10e98ad1

server/licenses/lucene-queryparser-7.4.0.jar.sha1

@@ -1 +0,0 @@
-8e58add0d0c39df97d07c8e343041989bf4b3a3f

server/licenses/lucene-queryparser-7.5.0-snapshot-608f0277b0.jar.sha1

@@ -0,0 +1 @@
+7ece30d5f1e18d96f61644451c858c3d9960558f

server/licenses/lucene-sandbox-7.4.0.jar.sha1

@@ -1 +0,0 @@
-1692604fa06a945d1ee19939022ef1a912235db3

server/licenses/lucene-sandbox-7.5.0-snapshot-608f0277b0.jar.sha1

@@ -0,0 +1 @@
+ad3bd0c2ed96556193c7215bef328e689d0b157f

server/licenses/lucene-spatial-7.4.0.jar.sha1

@@ -1 +0,0 @@
-847d2f897961124e2fc7d5e55d8309635bb026bc

server/licenses/lucene-spatial-7.5.0-snapshot-608f0277b0.jar.sha1

@@ -0,0 +1 @@
+8a6bd97e39ee5af60126adbe8c8375dc41b1ea8e

server/licenses/lucene-spatial-extras-7.4.0.jar.sha1

@@ -1 +0,0 @@
-586892eefc0546643d7f5d7f83659c7db0d534ff

server/licenses/lucene-spatial-extras-7.5.0-snapshot-608f0277b0.jar.sha1

@@ -0,0 +1 @@
+07e748d2d80000a7a213f3405b82b6e26b452948

server/licenses/lucene-spatial3d-7.4.0.jar.sha1

@@ -1 +0,0 @@
-32cd2854f39ff453a5d128ce40e11eea4168abbf

server/licenses/lucene-spatial3d-7.5.0-snapshot-608f0277b0.jar.sha1

@@ -0,0 +1 @@
+fd737bd5562f3943618ee7e73a0aaffb6319fdb2

server/licenses/lucene-suggest-7.4.0.jar.sha1

@@ -1 +0,0 @@
-0cdc1a512032f8b23dd4b1add0f5cd06325addc3

server/licenses/lucene-suggest-7.5.0-snapshot-608f0277b0.jar.sha1

@@ -0,0 +1 @@
+ff3f260d1dc8c18bc67f3c33aa84a0ad290daac5

server/src/main/java/org/elasticsearch/Version.java

@@ -182,7 +182,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_6_5_0 = new Version(V_6_5_0_ID, org.apache.lucene.util.Version.LUCENE_7_4_0);
     public static final int V_7_0_0_alpha1_ID = 7000001;
     public static final Version V_7_0_0_alpha1 =
-        new Version(V_7_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_7_4_0);
+        new Version(V_7_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_7_5_0);
     public static final Version CURRENT = V_7_0_0_alpha1;
 
     static {

server/src/main/java/org/elasticsearch/index/EsTieredMergePolicy.java

@@ -52,7 +52,7 @@ final class EsTieredMergePolicy extends FilterMergePolicy {
 
     @Override
     public MergeSpecification findForcedDeletesMerges(SegmentInfos infos, MergeContext mergeContext) throws IOException {
-        return forcedMergePolicy.findForcedDeletesMerges(infos, mergeContext);    
+        return forcedMergePolicy.findForcedDeletesMerges(infos, mergeContext);
     }
 
     public void setForceMergeDeletesPctAllowed(double forceMergeDeletesPctAllowed) {
@@ -108,13 +108,4 @@ final class EsTieredMergePolicy extends FilterMergePolicy {
     public double getSegmentsPerTier() {
         return regularMergePolicy.getSegmentsPerTier();
     }
-
-    public void setReclaimDeletesWeight(double reclaimDeletesWeight) {
-        regularMergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight);
-        forcedMergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight);
-    }
-
-    public double getReclaimDeletesWeight() {
-        return regularMergePolicy.getReclaimDeletesWeight();
-    }
 }

server/src/main/java/org/elasticsearch/index/IndexSettings.java

@@ -416,7 +416,6 @@ public final class IndexSettings {
         scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_EXPLICIT_SETTING, mergePolicyConfig::setMaxMergesAtOnceExplicit);
         scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING, mergePolicyConfig::setMaxMergedSegment);
         scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING, mergePolicyConfig::setSegmentsPerTier);
-        scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_RECLAIM_DELETES_WEIGHT_SETTING, mergePolicyConfig::setReclaimDeletesWeight);
 
         scopedSettings.addSettingsUpdateConsumer(MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING, MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING,
             mergeSchedulerConfig::setMaxThreadAndMergeCount);

server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java

@@ -150,7 +150,7 @@ public final class MergePolicyConfig {
             Property.Dynamic, Property.IndexScope);
     public static final Setting<Double> INDEX_MERGE_POLICY_RECLAIM_DELETES_WEIGHT_SETTING =
         Setting.doubleSetting("index.merge.policy.reclaim_deletes_weight", DEFAULT_RECLAIM_DELETES_WEIGHT, 0.0d,
-            Property.Dynamic, Property.IndexScope);
+            Property.Dynamic, Property.IndexScope, Property.Deprecated);
     public static final String INDEX_MERGE_ENABLED = "index.merge.enabled"; // don't convert to Setting<> and register... we only set this in tests and register via a plugin
 
 
@@ -176,17 +176,12 @@ public final class MergePolicyConfig {
         mergePolicy.setMaxMergeAtOnceExplicit(maxMergeAtOnceExplicit);
         mergePolicy.setMaxMergedSegmentMB(maxMergedSegment.getMbFrac());
         mergePolicy.setSegmentsPerTier(segmentsPerTier);
-        mergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight);
         if (logger.isTraceEnabled()) {
             logger.trace("using [tiered] merge mergePolicy with expunge_deletes_allowed[{}], floor_segment[{}], max_merge_at_once[{}], max_merge_at_once_explicit[{}], max_merged_segment[{}], segments_per_tier[{}], reclaim_deletes_weight[{}]",
                 forceMergeDeletesPctAllowed, floorSegment, maxMergeAtOnce, maxMergeAtOnceExplicit, maxMergedSegment, segmentsPerTier, reclaimDeletesWeight);
         }
     }
 
-    void setReclaimDeletesWeight(Double reclaimDeletesWeight) {
-        mergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight);
-    }
-
     void setSegmentsPerTier(Double segmentsPerTier) {
         mergePolicy.setSegmentsPerTier(segmentsPerTier);
     }

server/src/test/java/org/elasticsearch/index/EsTieredMergePolicyTests.java

@@ -29,16 +29,14 @@ public class EsTieredMergePolicyTests extends ESTestCase {
         assertEquals(
                 new TieredMergePolicy().getMaxMergedSegmentMB(),
                 policy.regularMergePolicy.getMaxMergedSegmentMB(), 0d);
-        // TODO: fix when incorporating https://issues.apache.org/jira/browse/LUCENE-8398, the first divisor must be a double
-        assertEquals(Long.MAX_VALUE / 1024 / 1024.0, policy.forcedMergePolicy.getMaxMergedSegmentMB(), 0d);
+        assertEquals(Long.MAX_VALUE / 1024.0 / 1024.0, policy.forcedMergePolicy.getMaxMergedSegmentMB(), 0d);
     }
 
     public void testSetMaxMergedSegmentMB() {
         EsTieredMergePolicy policy = new EsTieredMergePolicy();
         policy.setMaxMergedSegmentMB(10 * 1024);
         assertEquals(10 * 1024, policy.regularMergePolicy.getMaxMergedSegmentMB(), 0d);
-        // TODO: fix when incorporating https://issues.apache.org/jira/browse/LUCENE-8398, the first divisor must be a double
-        assertEquals(Long.MAX_VALUE / 1024 / 1024.0, policy.forcedMergePolicy.getMaxMergedSegmentMB(), 0d);
+        assertEquals(Long.MAX_VALUE / 1024.0 / 1024.0, policy.forcedMergePolicy.getMaxMergedSegmentMB(), 0d);
     }
 
     public void testSetForceMergeDeletesPctAllowed() {
@@ -71,10 +69,4 @@ public class EsTieredMergePolicyTests extends ESTestCase {
         policy.setSegmentsPerTier(42);
         assertEquals(42, policy.regularMergePolicy.getSegmentsPerTier(), 0);
     }
-
-    public void testSetReclaimDeletesWeight() {
-        EsTieredMergePolicy policy = new EsTieredMergePolicy();
-        policy.setReclaimDeletesWeight(42);
-        assertEquals(42, policy.regularMergePolicy.getReclaimDeletesWeight(), 0);
-    }
 }

server/src/test/java/org/elasticsearch/index/MergePolicySettingsTests.java

@@ -96,10 +96,6 @@ public class MergePolicySettingsTests extends ESTestCase {
         indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING.getKey(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1)).build()));
         assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001);
 
-        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT, 0);
-        indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_RECLAIM_DELETES_WEIGHT_SETTING.getKey(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT + 1).build()));
-        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT + 1, 0);
-
         assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER, 0);
         indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING.getKey(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER + 1).build()));
         assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER + 1, 0);
@@ -110,7 +106,6 @@ public class MergePolicySettingsTests extends ESTestCase {
         assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE);
         assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT);
         assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001);
-        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT, 0);
         assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER, 0);
     }
 

x-pack/plugin/sql/sql-action/licenses/lucene-core-7.4.0.jar.sha1

@@ -1 +0,0 @@
-730d9ac80436c8cbc0b2a8a749259be536b97316

x-pack/plugin/sql/sql-action/licenses/lucene-core-7.5.0-snapshot-608f0277b0.jar.sha1

@@ -0,0 +1 @@
+471096d6e92338b208aa91f3a85feb2f9cfc4afd