Upgrade to Lucene-8.9.0-SNAPSHOT-efdc43fee18 (#73130)

Upgrades to Lucene-8.9 snapshot which includes:

- LUCENE-9507: Custom order for leaves (/cc @mayya-sharipova)
- LUCENE-9935: Enable bulk merge for stored fields with index sort

buildSrc/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 8.0.0
-lucene            = 8.8.2
+lucene            = 8.9.0-snapshot-efdc43fee18
 
 bundled_jdk_vendor = adoptopenjdk
 bundled_jdk = 16.0.1+9

docs/Versions.asciidoc

@@ -1,8 +1,8 @@
 
 include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[]
 
-:lucene_version:        8.8.2
-:lucene_version_path:   8_8_2
+:lucene_version:        8.9.0
+:lucene_version_path:   8_9_0
 :jdk:                   11.0.2
 :jdk_major:             11
 :build_flavor:          default

modules/lang-expression/licenses/lucene-expressions-8.8.2.jar.sha1

@@ -1 +0,0 @@
-85c06b0d5b966e6d8a75dd568a6eb1a7c3b34bdd

modules/lang-expression/licenses/lucene-expressions-8.9.0-snapshot-efdc43fee18.jar.sha1

@@ -0,0 +1 @@
+a6b63a12259d51a1595d7e66131f79866ba5c77a

plugins/analysis-icu/licenses/lucene-analyzers-icu-8.8.2.jar.sha1

@@ -1 +0,0 @@
-dcda3ccd38ca68e23ec57c8ca16d13553066672b

plugins/analysis-icu/licenses/lucene-analyzers-icu-8.9.0-snapshot-efdc43fee18.jar.sha1

@@ -0,0 +1 @@
+ee8ece31d9801b3671ae91679a87fde7c94d94b1

plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.8.2.jar.sha1

@@ -1 +0,0 @@
-829b06a76ffaf0e551c5cda4a1e56990080ea6de

plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.9.0-snapshot-efdc43fee18.jar.sha1

@@ -0,0 +1 @@
+7964e1113fb0ab0b6ab063a78bae35af3b636731

plugins/analysis-nori/licenses/lucene-analyzers-nori-8.8.2.jar.sha1

@@ -1 +0,0 @@
-2c23e33c49fb255d96e0bf42ca7c58f030011727

plugins/analysis-nori/licenses/lucene-analyzers-nori-8.9.0-snapshot-efdc43fee18.jar.sha1

@@ -0,0 +1 @@
+c67a77f71bd9b3c853adade659c7e9a9192babcc

plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.8.2.jar.sha1

@@ -1 +0,0 @@
-b0ccf32c2baa88815b89c157f19b264a24958180

plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.9.0-snapshot-efdc43fee18.jar.sha1

@@ -0,0 +1 @@
+2a10e98d21375d09e46fe5435fbc61efe52d3759

plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.8.2.jar.sha1

@@ -1 +0,0 @@
-d7fb0c151dccc1954547acd3289cdf1369cda969

plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.9.0-snapshot-efdc43fee18.jar.sha1

@@ -0,0 +1 @@
+1452205f288857a3efd3cf9031fd826e3b7070a1

plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.8.2.jar.sha1

@@ -1 +0,0 @@
-ca1b3379c99553c64d6b76271a5701f2bc2a2ce4

plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.9.0-snapshot-efdc43fee18.jar.sha1

@@ -0,0 +1 @@
+9d0a2223b9bb63aab056884bdf63ef53451f7008

plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.8.2.jar.sha1

@@ -1 +0,0 @@
-2070b7dd21682b4b9bd23f77146a73d0e3f3b05f

plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.9.0-snapshot-efdc43fee18.jar.sha1

@@ -0,0 +1 @@
+1269c6ad244bcab76e213a8ae8978a03f43b48e3

server/licenses/lucene-analyzers-common-8.8.2.jar.sha1

@@ -1 +0,0 @@
-fa4b3c1d847f4bb91a3afaa990889dd164883d86

server/licenses/lucene-analyzers-common-8.9.0-snapshot-efdc43fee18.jar.sha1

@@ -0,0 +1 @@
+3957db6e9e79a25c1f87b928e9e131e90e1edaf3

server/licenses/lucene-backward-codecs-8.8.2.jar.sha1

@@ -1 +0,0 @@
-f4a0475a1404d7be87e911f5c85f8b913ec9770b

server/licenses/lucene-backward-codecs-8.9.0-snapshot-efdc43fee18.jar.sha1

@@ -0,0 +1 @@
+c2cc0334d0635406803aa3d34f0de9de202218c8

server/licenses/lucene-core-8.8.2.jar.sha1

@@ -1 +0,0 @@
-e039fa90ee900c475bbc0943989ed5dccc556d3e

server/licenses/lucene-core-8.9.0-snapshot-efdc43fee18.jar.sha1

@@ -0,0 +1 @@
+5112ca0a7c23fb4e727ee715a7027ba41b12c8bb

server/licenses/lucene-grouping-8.8.2.jar.sha1

@@ -1 +0,0 @@
-207ac15287656b6ce61ea921fefec3070a8dd99d

server/licenses/lucene-grouping-8.9.0-snapshot-efdc43fee18.jar.sha1

@@ -0,0 +1 @@
+98e484856b838e817488a9c4594adaa955e81c2d

server/licenses/lucene-highlighter-8.8.2.jar.sha1

@@ -1 +0,0 @@
-2674644580108335c08a25fd2e0f0c7dee010700

server/licenses/lucene-highlighter-8.9.0-snapshot-efdc43fee18.jar.sha1

@@ -0,0 +1 @@
+77f547f89f727b5ec2a4da19b268c90e999a87e8

server/licenses/lucene-join-8.8.2.jar.sha1

@@ -1 +0,0 @@
-6eb956741169fe8a5643fa39b14aef2fc067d865

server/licenses/lucene-join-8.9.0-snapshot-efdc43fee18.jar.sha1

@@ -0,0 +1 @@
+5cbb9006bfc710a5d8439bfda313ea7136dc2870

server/licenses/lucene-memory-8.8.2.jar.sha1

@@ -1 +0,0 @@
-63b6e8e862c898d263b941ffa55fe6c85bcdf2df

server/licenses/lucene-memory-8.9.0-snapshot-efdc43fee18.jar.sha1

@@ -0,0 +1 @@
+76724f40065fb6b0ad091eb62016a14ba9565fa4

server/licenses/lucene-misc-8.8.2.jar.sha1

@@ -1 +0,0 @@
-a3a916c27c42ff6773c52d0be4045d4e5c00f7f8

server/licenses/lucene-misc-8.9.0-snapshot-efdc43fee18.jar.sha1

@@ -0,0 +1 @@
+7fc790a6196e448d93842a55b9417a36ca6f103f

server/licenses/lucene-queries-8.8.2.jar.sha1

@@ -1 +0,0 @@
-1f324c3a6e1a7634bbada993b2c014d12d09cb83

server/licenses/lucene-queries-8.9.0-snapshot-efdc43fee18.jar.sha1

@@ -0,0 +1 @@
+3c83f2be4beaacfa6cacbd307d853847305a4ef8

server/licenses/lucene-queryparser-8.8.2.jar.sha1

@@ -1 +0,0 @@
-67c6580f8a5187fdb7f9eb2404bdecc4635b3328

server/licenses/lucene-queryparser-8.9.0-snapshot-efdc43fee18.jar.sha1

@@ -0,0 +1 @@
+ab8813f052ea06262ae09c4d7ffb347942a320f5

server/licenses/lucene-sandbox-8.8.2.jar.sha1

@@ -1 +0,0 @@
-b674deb4957a64db5c8f4afb9127400f4e65c334

server/licenses/lucene-sandbox-8.9.0-snapshot-efdc43fee18.jar.sha1

@@ -0,0 +1 @@
+33700525d6eab223e18f69f30050eb594084d04f

server/licenses/lucene-spatial-extras-8.8.2.jar.sha1

@@ -1 +0,0 @@
-227672b3b88c877b2df27608850ca15d834ec35c

server/licenses/lucene-spatial-extras-8.9.0-snapshot-efdc43fee18.jar.sha1

@@ -0,0 +1 @@
+a8ede8b7b1be8e9535363ec64d1f8534b0042d74

server/licenses/lucene-spatial3d-8.8.2.jar.sha1

@@ -1 +0,0 @@
-eb77b665a3b3e9ca6b3bb82c3342b843d9729d0d

server/licenses/lucene-spatial3d-8.9.0-snapshot-efdc43fee18.jar.sha1

@@ -0,0 +1 @@
+6cb14ba1a9485d27b9a0b45c9064f094aaa48c95

server/licenses/lucene-suggest-8.8.2.jar.sha1

@@ -1 +0,0 @@
-36fd5301e7143f0fcb0ba933f96148dc1e132b66

server/licenses/lucene-suggest-8.9.0-snapshot-efdc43fee18.jar.sha1

@@ -0,0 +1 @@
+a11a453cd1ec25ee2b429b01125e1e1d208b7f1c

server/src/main/java/org/elasticsearch/Version.java

@@ -83,7 +83,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_7_12_2 = new Version(7120299, org.apache.lucene.util.Version.LUCENE_8_8_0);
     public static final Version V_7_13_0 = new Version(7130099, org.apache.lucene.util.Version.LUCENE_8_8_2);
     public static final Version V_7_14_0 = new Version(7140099, org.apache.lucene.util.Version.LUCENE_8_8_2);
-    public static final Version V_8_0_0 = new Version(8000099, org.apache.lucene.util.Version.LUCENE_8_8_2);
+    public static final Version V_8_0_0 = new Version(8000099, org.apache.lucene.util.Version.LUCENE_8_9_0);
     public static final Version CURRENT = V_8_0_0;
 
     private static final ImmutableOpenIntMap<Version> idToVersion;

server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java

@@ -62,7 +62,7 @@ public final class NoOpEngine extends ReadOnlyEngine {
         final Directory directory = commit.getDirectory();
         final List<IndexCommit> indexCommits = DirectoryReader.listCommits(directory);
         final IndexCommit indexCommit = indexCommits.get(indexCommits.size() - 1);
-        return new DirectoryReader(directory, new LeafReader[0]) {
+        return new DirectoryReader(directory, new LeafReader[0], null) {
             @Override
             protected DirectoryReader doOpenIfChanged() {
                 return null;

server/src/main/java/org/elasticsearch/index/engine/SingleDocDirectoryReader.java

@@ -54,7 +54,7 @@ final class SingleDocDirectoryReader extends DirectoryReader {
     }
 
     private SingleDocDirectoryReader(SingleDocLeafReader leafReader) throws IOException {
-        super(leafReader.directory, new LeafReader[]{leafReader});
+        super(leafReader.directory, new LeafReader[]{leafReader}, null);
         this.leafReader = leafReader;
     }
 

test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java

@@ -183,7 +183,11 @@ public abstract class AnalysisFactoryTestCase extends ESTestCase {
         entry("concatenategraph", Void.class),
         // LUCENE-8936
         entry("spanishminimalstem", Void.class),
-        entry("delimitedboost", Void.class));
+        entry("delimitedboost", Void.class),
+        // LUCENE-9574
+        entry("dropifflagged", Void.class),
+        // LUCENE-9575
+        entry("patterntyping", Void.class));
 
     static final Map<String, Class<?>> KNOWN_CHARFILTERS = Map.of(
             "htmlstrip", MovedToAnalysisCommon.class,

x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java

@@ -69,7 +69,7 @@ public final class FrozenEngine extends ReadOnlyEngine {
             }
             this.docsStats = docsStats(reader);
             canMatchReader = ElasticsearchDirectoryReader.wrap(
-                new RewriteCachingDirectoryReader(directory, reader.leaves()), config.getShardId());
+                new RewriteCachingDirectoryReader(directory, reader.leaves(), null), config.getShardId());
             success = true;
         } catch (IOException e) {
             throw new UncheckedIOException(e);
@@ -85,7 +85,7 @@ public final class FrozenEngine extends ReadOnlyEngine {
         // we fake an empty DirectoryReader for the ReadOnlyEngine. this reader is only used
         // to initialize the reference manager and to make the refresh call happy which is essentially
         // a no-op now
-        return new DirectoryReader(indexCommit.getDirectory(), new LeafReader[0]) {
+        return new DirectoryReader(indexCommit.getDirectory(), new LeafReader[0], null) {
             @Override
             protected DirectoryReader doOpenIfChanged() {
                 return null;

x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/RewriteCachingDirectoryReader.java

@@ -27,6 +27,7 @@ import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.Bits;
 
 import java.io.IOException;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -39,9 +40,9 @@ import java.util.Map;
  */
 final class RewriteCachingDirectoryReader extends DirectoryReader {
 
-    RewriteCachingDirectoryReader(Directory directory, List<LeafReaderContext> segmentReaders) throws
-        IOException {
-        super(directory, wrap(segmentReaders));
+    RewriteCachingDirectoryReader(Directory directory, List<LeafReaderContext> segmentReaders,
+                                  Comparator<LeafReader> leafSorter) throws IOException {
+        super(directory, wrap(segmentReaders), leafSorter);
     }
 
     private static LeafReader[] wrap(List<LeafReaderContext> readers) throws IOException {

x-pack/plugin/frozen-indices/src/test/java/org/elasticsearch/index/engine/RewriteCachingDirectoryReaderTests.java

@@ -40,7 +40,7 @@ public class RewriteCachingDirectoryReaderTests extends ESTestCase {
                     writer.addDocument(doc);
                 }
                 try (DirectoryReader reader = DirectoryReader.open(writer)) {
-                    RewriteCachingDirectoryReader cachingDirectoryReader = new RewriteCachingDirectoryReader(dir, reader.leaves());
+                    RewriteCachingDirectoryReader cachingDirectoryReader = new RewriteCachingDirectoryReader(dir, reader.leaves(), null);
                     if (rarely) {
                         assertArrayEquals(PointValues.getMaxPackedValue(reader, "rarely"),
                             PointValues.getMaxPackedValue(cachingDirectoryReader, "rarely"));
@@ -87,7 +87,7 @@ public class RewriteCachingDirectoryReaderTests extends ESTestCase {
                 doc.add(new LongPoint("test", 10));
                 writer.addDocument(doc);
                 try (DirectoryReader reader = DirectoryReader.open(writer)) {
-                    RewriteCachingDirectoryReader cachingDirectoryReader = new RewriteCachingDirectoryReader(dir, reader.leaves());
+                    RewriteCachingDirectoryReader cachingDirectoryReader = new RewriteCachingDirectoryReader(dir, reader.leaves(), null);
                     DateFieldMapper.DateFieldType dateFieldType = new DateFieldMapper.DateFieldType("test");
                     QueryRewriteContext context = new QueryRewriteContext(xContentRegistry(), writableRegistry(), null, () -> 0);
                     MappedFieldType.Relation relation = dateFieldType.isFieldWithinQuery(cachingDirectoryReader, 0, 10,

x-pack/plugin/sql/sql-action/licenses/lucene-core-8.8.2.jar.sha1

@@ -1 +0,0 @@
-e039fa90ee900c475bbc0943989ed5dccc556d3e

x-pack/plugin/sql/sql-action/licenses/lucene-core-8.9.0-snapshot-efdc43fee18.jar.sha1

@@ -0,0 +1 @@
+5112ca0a7c23fb4e727ee715a7027ba41b12c8bb