Upgrade to new lucene snapshot 9.7.0-snapshot-41cd1f7a88c (#96741)

Notable changes:

* more efficient backwards reads in NIOFSDirectory
* faster merging when using soft deletes
* workaround security manager when using vector API

build-tools-internal/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 8.9.0
-lucene            = 9.7.0-snapshot-24df30cca69
+lucene            = 9.7.0-snapshot-41cd1f7a88c
 
 bundled_jdk_vendor = openjdk
 bundled_jdk = 20.0.1+9@b4887098932d415489976708ad6d1a4b

docs/changelog/96741.yaml

@@ -0,0 +1,5 @@
+pr: 96741
+summary: Upgrade to new lucene snapshot 9.7.0-snapshot-a8602d6ef88
+area: Search
+type: upgrade
+issues: []

gradle/verification-metadata.xml

@@ -2483,124 +2483,124 @@
             <sha256 value="015d5c229f3cd5c0ebf175c1da08d596d94043362ae9d92637d88848c90537c8" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-analysis-common" version="9.7.0-snapshot-24df30cca69">
-         <artifact name="lucene-analysis-common-9.7.0-snapshot-24df30cca69.jar">
-            <sha256 value="5c5a049f51a233d2a3f0c50bdb625b52905c1c537b3e38e040e6517bfa074b2b" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-analysis-common" version="9.7.0-snapshot-41cd1f7a88c">
+         <artifact name="lucene-analysis-common-9.7.0-snapshot-41cd1f7a88c.jar">
+            <sha256 value="dc54c36b1a6e51d6a39f70e760570d1aaa17558980b09a4aab06da64f932dd62" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-analysis-icu" version="9.7.0-snapshot-24df30cca69">
-         <artifact name="lucene-analysis-icu-9.7.0-snapshot-24df30cca69.jar">
-            <sha256 value="6e378b9a818699e6fcb3cdda4187a4672bfa47b2de288a3c8f258b243954f476" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-analysis-icu" version="9.7.0-snapshot-41cd1f7a88c">
+         <artifact name="lucene-analysis-icu-9.7.0-snapshot-41cd1f7a88c.jar">
+            <sha256 value="184899387f830c370e4c2814eb11cd4cd630f6257fab3db56fa21d55ac845adf" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-analysis-kuromoji" version="9.7.0-snapshot-24df30cca69">
-         <artifact name="lucene-analysis-kuromoji-9.7.0-snapshot-24df30cca69.jar">
-            <sha256 value="b05149a2feae54626e870a7829e8b00245f59be5985809e1c55830e5b2f219cc" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-analysis-kuromoji" version="9.7.0-snapshot-41cd1f7a88c">
+         <artifact name="lucene-analysis-kuromoji-9.7.0-snapshot-41cd1f7a88c.jar">
+            <sha256 value="6233a2a517a748522fe44683778dc0566f2d2a427203a092fdb842154b545662" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-analysis-morfologik" version="9.7.0-snapshot-24df30cca69">
-         <artifact name="lucene-analysis-morfologik-9.7.0-snapshot-24df30cca69.jar">
-            <sha256 value="aa340253882a997370ddb3a0350f800d48386e496e9362834dbe8dd2eb86e385" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-analysis-morfologik" version="9.7.0-snapshot-41cd1f7a88c">
+         <artifact name="lucene-analysis-morfologik-9.7.0-snapshot-41cd1f7a88c.jar">
+            <sha256 value="103359e987ecc2b3f119bdf891428bc6a54f4371cb730a6b6c83aaf73119eec5" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-analysis-nori" version="9.7.0-snapshot-24df30cca69">
-         <artifact name="lucene-analysis-nori-9.7.0-snapshot-24df30cca69.jar">
-            <sha256 value="88259a45357e7ae085cddd3da37352839d0bcb8b3b7f99acc7065859d851ff06" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-analysis-nori" version="9.7.0-snapshot-41cd1f7a88c">
+         <artifact name="lucene-analysis-nori-9.7.0-snapshot-41cd1f7a88c.jar">
+            <sha256 value="83ac1452b550571171d321adb1fad4e7da1a29d7408346825e3137bad84e12f5" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-analysis-phonetic" version="9.7.0-snapshot-24df30cca69">
-         <artifact name="lucene-analysis-phonetic-9.7.0-snapshot-24df30cca69.jar">
-            <sha256 value="2dae3e41c203e8049f792e9328f3d291ffed42a0c6dc422e92d0f4b785759194" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-analysis-phonetic" version="9.7.0-snapshot-41cd1f7a88c">
+         <artifact name="lucene-analysis-phonetic-9.7.0-snapshot-41cd1f7a88c.jar">
+            <sha256 value="5e09eefd06c708604994c142fd0a1ee52315d02aa318bf3cbc25771e247a8f67" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-analysis-smartcn" version="9.7.0-snapshot-24df30cca69">
-         <artifact name="lucene-analysis-smartcn-9.7.0-snapshot-24df30cca69.jar">
-            <sha256 value="67ca1d76a228ef9b3d7ecf4522c250f30567c016fb3e352df54d74bcffcf6e95" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-analysis-smartcn" version="9.7.0-snapshot-41cd1f7a88c">
+         <artifact name="lucene-analysis-smartcn-9.7.0-snapshot-41cd1f7a88c.jar">
+            <sha256 value="3c28f3d2daa5e467ee44efb1b345f2aacffc790b6af40ae577ccc941fcddc735" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-analysis-stempel" version="9.7.0-snapshot-24df30cca69">
-         <artifact name="lucene-analysis-stempel-9.7.0-snapshot-24df30cca69.jar">
-            <sha256 value="d3bafcd84a883451c24731da5144bf018fa819e909cb05776635a7ba11940826" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-analysis-stempel" version="9.7.0-snapshot-41cd1f7a88c">
+         <artifact name="lucene-analysis-stempel-9.7.0-snapshot-41cd1f7a88c.jar">
+            <sha256 value="38674e2413c6097a1557abb18ed88aa9fd41624ccecd9d301cdf4bacfd3bff97" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-backward-codecs" version="9.7.0-snapshot-24df30cca69">
-         <artifact name="lucene-backward-codecs-9.7.0-snapshot-24df30cca69.jar">
-            <sha256 value="4ae23e18f3ee52ae0eaceffc70daa62023fdc2cc37896408e68c0b5969235167" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-backward-codecs" version="9.7.0-snapshot-41cd1f7a88c">
+         <artifact name="lucene-backward-codecs-9.7.0-snapshot-41cd1f7a88c.jar">
+            <sha256 value="858c1f5bc5403d35f4783e3de50058d54ebce24eba13e61705c0b898ababc801" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-codecs" version="9.7.0-snapshot-24df30cca69">
-         <artifact name="lucene-codecs-9.7.0-snapshot-24df30cca69.jar">
-            <sha256 value="256d0c240e96d645d872de342cbcfda3a8ee55e4ee34312d4d2214cf0825f70e" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-codecs" version="9.7.0-snapshot-41cd1f7a88c">
+         <artifact name="lucene-codecs-9.7.0-snapshot-41cd1f7a88c.jar">
+            <sha256 value="c90eae3399f31f5b5b70590dbd4e2d444c772cb93a64414b9a56564efa424bb5" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-core" version="9.7.0-snapshot-24df30cca69">
-         <artifact name="lucene-core-9.7.0-snapshot-24df30cca69.jar">
-            <sha256 value="847a6870085fbe3ffb1b61291d1555c1a3b3e4f9f1fc84bb311527822134b93b" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-core" version="9.7.0-snapshot-41cd1f7a88c">
+         <artifact name="lucene-core-9.7.0-snapshot-41cd1f7a88c.jar">
+            <sha256 value="334596f8dd6e5d03624d8ac31a9f024c65358a50ca6810bbbaedc8841481ee46" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-expressions" version="9.7.0-snapshot-24df30cca69">
-         <artifact name="lucene-expressions-9.7.0-snapshot-24df30cca69.jar">
-            <sha256 value="31c06faa38ddd9f529f81b61c789f926c7f19f198ae6eaa13a84c1cc36f94530" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-expressions" version="9.7.0-snapshot-41cd1f7a88c">
+         <artifact name="lucene-expressions-9.7.0-snapshot-41cd1f7a88c.jar">
+            <sha256 value="b4048f71261d04ac795ca19e607e74e3c733912247b037e00cebcee9204c0df4" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-grouping" version="9.7.0-snapshot-24df30cca69">
-         <artifact name="lucene-grouping-9.7.0-snapshot-24df30cca69.jar">
-            <sha256 value="a5f24c0a842575faadb001ad6651fa59c611dc3cba70903d27243ebfc58c264f" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-grouping" version="9.7.0-snapshot-41cd1f7a88c">
+         <artifact name="lucene-grouping-9.7.0-snapshot-41cd1f7a88c.jar">
+            <sha256 value="e5710ec50aa5ea1414141c17fe10e87e4c5980d66890f17a61ddd6eaef37cf4c" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-highlighter" version="9.7.0-snapshot-24df30cca69">
-         <artifact name="lucene-highlighter-9.7.0-snapshot-24df30cca69.jar">
-            <sha256 value="96c1ceeae0ec2e3a5764f196695c56833b737716235fa51bd55c1fbc07621890" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-highlighter" version="9.7.0-snapshot-41cd1f7a88c">
+         <artifact name="lucene-highlighter-9.7.0-snapshot-41cd1f7a88c.jar">
+            <sha256 value="50d98c27c521cb40642e41903dc20bb0fae61159528b7d3d503726e2326c4496" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-join" version="9.7.0-snapshot-24df30cca69">
-         <artifact name="lucene-join-9.7.0-snapshot-24df30cca69.jar">
-            <sha256 value="792e44cf559ebd0c7173ec6947c0305643767504043c30086c17aeea94e3adc5" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-join" version="9.7.0-snapshot-41cd1f7a88c">
+         <artifact name="lucene-join-9.7.0-snapshot-41cd1f7a88c.jar">
+            <sha256 value="8174a381295deba79dca2bc70e11e895a558e04314db6ad7ffa4190fc450c67e" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-memory" version="9.7.0-snapshot-24df30cca69">
-         <artifact name="lucene-memory-9.7.0-snapshot-24df30cca69.jar">
-            <sha256 value="08b97f8a31b002d3805bcc7aa3bfd007c0c3c3a15637d85ed0c2661a39a1b8ed" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-memory" version="9.7.0-snapshot-41cd1f7a88c">
+         <artifact name="lucene-memory-9.7.0-snapshot-41cd1f7a88c.jar">
+            <sha256 value="687f3ac2d9586c3d5d1c1a12d296265a1e2f6fe62e6f3110eb63306b426d83c5" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-misc" version="9.7.0-snapshot-24df30cca69">
-         <artifact name="lucene-misc-9.7.0-snapshot-24df30cca69.jar">
-            <sha256 value="75efe101120e3f762ba7924373d489ca95c67cb976ef34967b599c1145cee3ec" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-misc" version="9.7.0-snapshot-41cd1f7a88c">
+         <artifact name="lucene-misc-9.7.0-snapshot-41cd1f7a88c.jar">
+            <sha256 value="cce3e0cb150c5b5223a3643749858b5093ee0ecbb410d52d25d5aed161f7e2c0" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-queries" version="9.7.0-snapshot-24df30cca69">
-         <artifact name="lucene-queries-9.7.0-snapshot-24df30cca69.jar">
-            <sha256 value="3396de8adf2932902c1f41a4c5c0ac60864d837d978e11a08387ecd05ae43533" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-queries" version="9.7.0-snapshot-41cd1f7a88c">
+         <artifact name="lucene-queries-9.7.0-snapshot-41cd1f7a88c.jar">
+            <sha256 value="53f64d898bd8563e4ff102de79fe0d1b4589a678f1155bf2aab98cb38a71385e" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-queryparser" version="9.7.0-snapshot-24df30cca69">
-         <artifact name="lucene-queryparser-9.7.0-snapshot-24df30cca69.jar">
-            <sha256 value="2c66472d5c93c2681f15f46e366dc2359e0054e3745e7ea10b2a2cb69996447a" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-queryparser" version="9.7.0-snapshot-41cd1f7a88c">
+         <artifact name="lucene-queryparser-9.7.0-snapshot-41cd1f7a88c.jar">
+            <sha256 value="7f6d2fbcc4522a9dcb57006cd753907a3439023add3a86586942c60e2b032b67" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-sandbox" version="9.7.0-snapshot-24df30cca69">
-         <artifact name="lucene-sandbox-9.7.0-snapshot-24df30cca69.jar">
-            <sha256 value="56986817350e5b415130316c794dcc16b73f9dd835af311c3a1d28d1f5e86895" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-sandbox" version="9.7.0-snapshot-41cd1f7a88c">
+         <artifact name="lucene-sandbox-9.7.0-snapshot-41cd1f7a88c.jar">
+            <sha256 value="c34508a739d481a787df484bfa056f7f02a4c037fee93ee89337f6fe639d01da" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-spatial-extras" version="9.7.0-snapshot-24df30cca69">
-         <artifact name="lucene-spatial-extras-9.7.0-snapshot-24df30cca69.jar">
-            <sha256 value="fbb2be4c0e10c12903ceb5cf6cc931c07c9f331d2f247ca975781235004a4693" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-spatial-extras" version="9.7.0-snapshot-41cd1f7a88c">
+         <artifact name="lucene-spatial-extras-9.7.0-snapshot-41cd1f7a88c.jar">
+            <sha256 value="30933d9265ec2bb1022cad1320a332be4e42289bffd83fdd8630539f84806387" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-spatial3d" version="9.7.0-snapshot-24df30cca69">
-         <artifact name="lucene-spatial3d-9.7.0-snapshot-24df30cca69.jar">
-            <sha256 value="16eb558e5120bcf9c609c8a8f5083176ed260f8af31366a1bbcb9521c78608fa" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-spatial3d" version="9.7.0-snapshot-41cd1f7a88c">
+         <artifact name="lucene-spatial3d-9.7.0-snapshot-41cd1f7a88c.jar">
+            <sha256 value="f85895acf2fefc38db65538ad11e0bba9e30a23dc36a5b3e630c9a5da6e799d4" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-suggest" version="9.7.0-snapshot-24df30cca69">
-         <artifact name="lucene-suggest-9.7.0-snapshot-24df30cca69.jar">
-            <sha256 value="74b766301bedd21579b473a6a65b280ae5408b7b8144adb5f29d2520494bd40e" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-suggest" version="9.7.0-snapshot-41cd1f7a88c">
+         <artifact name="lucene-suggest-9.7.0-snapshot-41cd1f7a88c.jar">
+            <sha256 value="0bc82ca849ead94f9b2f1981c6a2c6c3737172a3219e9425a4ca6baf3ba33632" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-test-framework" version="9.7.0-snapshot-24df30cca69">
-         <artifact name="lucene-test-framework-9.7.0-snapshot-24df30cca69.jar">
-            <sha256 value="0e93af73fc5bd1c836fbe45c2050161064c17df5d2e1c0b5b014280d7647deaa" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-test-framework" version="9.7.0-snapshot-41cd1f7a88c">
+         <artifact name="lucene-test-framework-9.7.0-snapshot-41cd1f7a88c.jar">
+            <sha256 value="d330f1971e52fca545da2dcb44e00e1702e2c47f8448cbce5f0230c1a7b085c5" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="org.apache.maven" name="maven-model" version="3.6.2">

modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java

@@ -137,6 +137,7 @@ public class CommonAnalysisFactoryTests extends AnalysisFactoryTestCase {
         filters.put("germanstem", GermanStemTokenFilterFactory.class);
         // this filter is not exposed and should only be used internally
         filters.put("fixedshingle", Void.class);
+        filters.put("word2vecsynonym", Void.class); // not exposed
         return filters;
     }
 

server/src/main/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilder.java

@@ -21,6 +21,7 @@ import org.apache.lucene.search.similarities.BM25Similarity;
 import org.apache.lucene.search.similarities.Similarity;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.QueryBuilder;
+import org.apache.lucene.util.TermAndBoost;
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;

server/src/main/java/org/elasticsearch/index/search/MatchQueryParser.java

@@ -28,6 +28,7 @@ import org.apache.lucene.search.FuzzyQuery;
 import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.QueryBuilder;
+import org.apache.lucene.util.TermAndBoost;
 import org.apache.lucene.util.graph.GraphTokenStreamFiniteStrings;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.io.stream.StreamInput;

server/src/main/java/org/elasticsearch/index/search/MultiMatchQueryParser.java

@@ -18,6 +18,7 @@ import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.TermAndBoost;
 import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.query.AbstractQueryBuilder;

test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java

@@ -195,7 +195,9 @@ public abstract class AnalysisFactoryTestCase extends ESTestCase {
         entry("spanishpluralstem", Void.class),
         // LUCENE-10352
         entry("daitchmokotoffsoundex", Void.class),
-        entry("persianstem", Void.class)
+        entry("persianstem", Void.class),
+        // not exposed
+        entry("word2vecsynonym", Void.class)
     );
 
     static final Map<String, Class<?>> KNOWN_CHARFILTERS = Map.of(