Upgrade to a new lucene 8.8.0 snapshot (#67691)

This change upgrades to the latest Lucene 8.8.0 snapshot.
It also restores the compression on binary doc values that was lost in the last snapshot upgrade.
The compression is now configurable on binary doc values but we don't expose this functionality yet so this commit ensures that we pick the same compression mode as previous releases (BEST_COMPRESSION).

buildSrc/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 8.0.0
-lucene            = 8.8.0-snapshot-f73f6b1
+lucene            = 8.8.0-snapshot-737cb9c49b0
 
 bundled_jdk_vendor = adoptopenjdk
 bundled_jdk = 15.0.1+9

modules/lang-expression/licenses/lucene-expressions-8.8.0-snapshot-737cb9c49b0.jar.sha1

@@ -0,0 +1 @@
+b4f3adb6e7df5147c448505aba864be673b9f164

modules/lang-expression/licenses/lucene-expressions-8.8.0-snapshot-f73f6b1.jar.sha1

@@ -1 +0,0 @@
-6d82d0d63473800a6f8dd8f67cfa9e4946904e30

plugins/analysis-icu/licenses/lucene-analyzers-icu-8.8.0-snapshot-737cb9c49b0.jar.sha1

@@ -0,0 +1 @@
+f5f7216797e140f4aa66236bbbf148e097fcce89

plugins/analysis-icu/licenses/lucene-analyzers-icu-8.8.0-snapshot-f73f6b1.jar.sha1

@@ -1 +0,0 @@
-f8fecb3a1fb8f52965708ea94b1eba9449f777f6

plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.8.0-snapshot-737cb9c49b0.jar.sha1

@@ -0,0 +1 @@
+5ebf9236135170da29a2c0afac99802d245ce4b1

plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.8.0-snapshot-f73f6b1.jar.sha1

@@ -1 +0,0 @@
-78cdebe4a4066f7e52a999fad285ec8a9e4ec0e0

plugins/analysis-nori/licenses/lucene-analyzers-nori-8.8.0-snapshot-737cb9c49b0.jar.sha1

@@ -0,0 +1 @@
+7ed1c02e826b26bbf1e149f4c79597bf38d777af

plugins/analysis-nori/licenses/lucene-analyzers-nori-8.8.0-snapshot-f73f6b1.jar.sha1

@@ -1 +0,0 @@
-0015db2056dea6ffdf70930daf830010510ee105

plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.8.0-snapshot-737cb9c49b0.jar.sha1

@@ -0,0 +1 @@
+49f77a41f4cb532bd8f19072da4e563eef234749

plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.8.0-snapshot-f73f6b1.jar.sha1

@@ -1 +0,0 @@
-ddf560f4ebbe9a48b263334c23f61ce2449bc67c

plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.8.0-snapshot-737cb9c49b0.jar.sha1

@@ -0,0 +1 @@
+5a3f3a091a12bd9cecb6884f6a17eb0d98b90f26

plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.8.0-snapshot-f73f6b1.jar.sha1

@@ -1 +0,0 @@
-ac0e338dfb5c2b3ea19434f612c58063bfaf860e

plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.8.0-snapshot-737cb9c49b0.jar.sha1

@@ -0,0 +1 @@
+c22909f31d7f8ff4438a791b84948dac68a49757

plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.8.0-snapshot-f73f6b1.jar.sha1

@@ -1 +0,0 @@
-ffbdf7991e01cb0a68869c81aa093afe94a9f221

plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.8.0-snapshot-737cb9c49b0.jar.sha1

@@ -0,0 +1 @@
+93087fba7581472d49b32895f164220f0c89877f

plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.8.0-snapshot-f73f6b1.jar.sha1

@@ -1 +0,0 @@
-eee0bfa4c86c5849584f1be0b80c8f6a81d33943

server/licenses/lucene-analyzers-common-8.8.0-snapshot-737cb9c49b0.jar.sha1

@@ -0,0 +1 @@
+dec36ca8de56e0bc788654e0318289a8974073a2

server/licenses/lucene-analyzers-common-8.8.0-snapshot-f73f6b1.jar.sha1

@@ -1 +0,0 @@
-8b81a0d1a5f52dc6f4a3f35b3d199d0b6a312cc0

server/licenses/lucene-backward-codecs-8.8.0-snapshot-737cb9c49b0.jar.sha1

@@ -0,0 +1 @@
+f375721f0f91f6681444c86d8c653db0e713328b

server/licenses/lucene-backward-codecs-8.8.0-snapshot-f73f6b1.jar.sha1

@@ -1 +0,0 @@
-48d676f90a23fdd0b91a0ff220987d7d519ed10f

server/licenses/lucene-core-8.8.0-snapshot-737cb9c49b0.jar.sha1

@@ -0,0 +1 @@
+8e6ecdfc54e2755403c932b3061e155afd6dce35

server/licenses/lucene-core-8.8.0-snapshot-f73f6b1.jar.sha1

@@ -1 +0,0 @@
-3e597bad519d7064167d795e4395432c5957077e

server/licenses/lucene-grouping-8.8.0-snapshot-737cb9c49b0.jar.sha1

@@ -0,0 +1 @@
+23594b7cf1d5a9fb337b9f2274789c57cc83c4ac

server/licenses/lucene-grouping-8.8.0-snapshot-f73f6b1.jar.sha1

@@ -1 +0,0 @@
-65935fa078f32fe4ddcf2f22a74019d5272dd8c7

server/licenses/lucene-highlighter-8.8.0-snapshot-737cb9c49b0.jar.sha1

@@ -0,0 +1 @@
+4303ec3c9f6dc214a253c72a56d8b0dac6cc6860

server/licenses/lucene-highlighter-8.8.0-snapshot-f73f6b1.jar.sha1

@@ -1 +0,0 @@
-c9bfea368e9d7247b5b6bc19be3ae4f3f8232935

server/licenses/lucene-join-8.8.0-snapshot-737cb9c49b0.jar.sha1

@@ -0,0 +1 @@
+ac954190443061cf16e4850ba28beeaa05fe3d45

server/licenses/lucene-join-8.8.0-snapshot-f73f6b1.jar.sha1

@@ -1 +0,0 @@
-cc8c2b7fcb679fcef8511ecdcec824605bd88e59

server/licenses/lucene-memory-8.8.0-snapshot-737cb9c49b0.jar.sha1

@@ -0,0 +1 @@
+9964c3f672aa308c6f6127cbe544565e937035fa

server/licenses/lucene-memory-8.8.0-snapshot-f73f6b1.jar.sha1

@@ -1 +0,0 @@
-2e9d039db4b640860420cdf288e2e23b93b06082

server/licenses/lucene-misc-8.8.0-snapshot-737cb9c49b0.jar.sha1

@@ -0,0 +1 @@
+05879a6ae1b5b2b24ed6d35f1e02d828e0c3acda

server/licenses/lucene-misc-8.8.0-snapshot-f73f6b1.jar.sha1

@@ -1 +0,0 @@
-3825d3b91339a5daf57fb0dd6d5f5a475ae373a0

server/licenses/lucene-queries-8.8.0-snapshot-737cb9c49b0.jar.sha1

@@ -0,0 +1 @@
+4d35878e64f472e8e526c68770338ded1febad0b

server/licenses/lucene-queries-8.8.0-snapshot-f73f6b1.jar.sha1

@@ -1 +0,0 @@
-54fd24b7d35e191909bc7110f2984f6301bcc8dd

server/licenses/lucene-queryparser-8.8.0-snapshot-737cb9c49b0.jar.sha1

@@ -0,0 +1 @@
+6c2d0701fcf908b122f5a691fb7c43782ac98ce3

server/licenses/lucene-queryparser-8.8.0-snapshot-f73f6b1.jar.sha1

@@ -1 +0,0 @@
-1c314875e90073b3d140bafc534895c587618380

server/licenses/lucene-sandbox-8.8.0-snapshot-737cb9c49b0.jar.sha1

@@ -0,0 +1 @@
+f090f5416210d29c8489c5fdbd592734bf3eb482

server/licenses/lucene-sandbox-8.8.0-snapshot-f73f6b1.jar.sha1

@@ -1 +0,0 @@
-c4bb882ae7892a0683aa977617722065338ea1ce

server/licenses/lucene-spatial-extras-8.8.0-snapshot-737cb9c49b0.jar.sha1

@@ -0,0 +1 @@
+86cfe9c23413b2710149d0281d28980990971a97

server/licenses/lucene-spatial-extras-8.8.0-snapshot-f73f6b1.jar.sha1

@@ -1 +0,0 @@
-84ff8828cfb2b623b8e38a5535897308f67cf9c2

server/licenses/lucene-spatial3d-8.8.0-snapshot-737cb9c49b0.jar.sha1

@@ -0,0 +1 @@
+9c423aedc7e3e2097695f414951b46239eb6971a

server/licenses/lucene-spatial3d-8.8.0-snapshot-f73f6b1.jar.sha1

@@ -1 +0,0 @@
-cbda3f9940ab327fa7bf8181ed26fca9c0a11436

server/licenses/lucene-suggest-8.8.0-snapshot-737cb9c49b0.jar.sha1

@@ -0,0 +1 @@
+1902e7bdfbb7829ba2c4df5b37f1511871a2eeed

server/licenses/lucene-suggest-8.8.0-snapshot-f73f6b1.jar.sha1

@@ -1 +0,0 @@
-7fb7f6451cde1418acfa6291c1293bb97a23ded7

server/src/main/java/org/elasticsearch/index/codec/PerFieldMappingPostingFormatCodec.java

@@ -21,7 +21,9 @@ package org.elasticsearch.index.codec;
 
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.PostingsFormat;
+import org.apache.lucene.codecs.lucene80.Lucene80DocValuesFormat;
 import org.apache.lucene.codecs.lucene87.Lucene87Codec;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.index.mapper.CompletionFieldMapper;
@@ -39,6 +41,8 @@ import org.elasticsearch.index.mapper.MapperService;
 public class PerFieldMappingPostingFormatCodec extends Lucene87Codec {
     private final Logger logger;
     private final MapperService mapperService;
+    // Always enable compression on binary doc values
+    private final DocValuesFormat docValuesFormat = new Lucene80DocValuesFormat(Lucene80DocValuesFormat.Mode.BEST_COMPRESSION);
 
     static {
         assert Codec.forName(Lucene.LATEST_CODEC).getClass().isAssignableFrom(PerFieldMappingPostingFormatCodec.class) :
@@ -62,4 +66,8 @@ public class PerFieldMappingPostingFormatCodec extends Lucene87Codec {
         return super.getPostingsFormatForField(field);
     }
 
+    @Override
+    public DocValuesFormat getDocValuesFormatForField(String field) {
+        return docValuesFormat;
+    }
 }

server/src/test/java/org/elasticsearch/index/codec/CodecTests.java

@@ -21,15 +21,17 @@ package org.elasticsearch.index.codec;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.lucene80.Lucene80DocValuesFormat;
 import org.apache.lucene.codecs.lucene87.Lucene87Codec;
 import org.apache.lucene.codecs.lucene87.Lucene87StoredFieldsFormat;
-import org.apache.lucene.codecs.lucene87.Lucene87StoredFieldsFormat.Mode;
+import org.apache.lucene.document.BinaryDocValuesField;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.SegmentReader;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
@@ -59,16 +61,18 @@ public class CodecTests extends ESTestCase {
 
     public void testDefault() throws Exception {
         Codec codec = createCodecService().codec("default");
-        assertCompressionEquals(Mode.BEST_SPEED, codec);
+        assertStoredFieldsCompressionEquals(Lucene87StoredFieldsFormat.Mode.BEST_SPEED, codec);
+        assertDocValuesCompressionEquals(Lucene80DocValuesFormat.Mode.BEST_COMPRESSION, codec);
     }
 
     public void testBestCompression() throws Exception {
         Codec codec = createCodecService().codec("best_compression");
-        assertCompressionEquals(Mode.BEST_COMPRESSION, codec);
+        assertStoredFieldsCompressionEquals(Lucene87StoredFieldsFormat.Mode.BEST_COMPRESSION, codec);
+        assertDocValuesCompressionEquals(Lucene80DocValuesFormat.Mode.BEST_COMPRESSION, codec);
     }
 
     // write some docs with it, inspect .si to see this was the used compression
-    private void assertCompressionEquals(Mode expected, Codec actual) throws Exception {
+    private void assertStoredFieldsCompressionEquals(Lucene87StoredFieldsFormat.Mode expected, Codec actual) throws Exception {
         Directory dir = newDirectory();
         IndexWriterConfig iwc = newIndexWriterConfig(null);
         iwc.setCodec(actual);
@@ -80,7 +84,27 @@ public class CodecTests extends ESTestCase {
         SegmentReader sr = (SegmentReader) ir.leaves().get(0).reader();
         String v = sr.getSegmentInfo().info.getAttribute(Lucene87StoredFieldsFormat.MODE_KEY);
         assertNotNull(v);
-        assertEquals(expected, Mode.valueOf(v));
+        assertEquals(expected, Lucene87StoredFieldsFormat.Mode.valueOf(v));
+        ir.close();
+        dir.close();
+    }
+
+    // write some docs with it, inspect .si to see this was the used compression
+    private void assertDocValuesCompressionEquals(Lucene80DocValuesFormat.Mode expected, Codec actual) throws Exception {
+        Directory dir = newDirectory();
+        IndexWriterConfig iwc = newIndexWriterConfig(null);
+        iwc.setCodec(actual);
+        IndexWriter iw = new IndexWriter(dir, iwc);
+        Document doc = new Document();
+        doc.add(new BinaryDocValuesField("foo", new BytesRef("aaa")));
+        iw.addDocument(doc);
+        iw.commit();
+        iw.close();
+        DirectoryReader ir = DirectoryReader.open(dir);
+        SegmentReader sr = (SegmentReader) ir.leaves().get(0).reader();
+        String v = sr.getFieldInfos().fieldInfo("foo").getAttribute(Lucene80DocValuesFormat.MODE_KEY);
+        assertNotNull(v);
+        assertEquals(expected, Lucene80DocValuesFormat.Mode.valueOf(v));
         ir.close();
         dir.close();
     }

x-pack/plugin/sql/sql-action/licenses/lucene-core-8.8.0-snapshot-737cb9c49b0.jar.sha1

@@ -0,0 +1 @@
+8e6ecdfc54e2755403c932b3061e155afd6dce35

x-pack/plugin/sql/sql-action/licenses/lucene-core-8.8.0-snapshot-f73f6b1.jar.sha1

@@ -1 +0,0 @@
-3e597bad519d7064167d795e4395432c5957077e