Upgrade to lucene 8.0.0-snapshot-ff9509a8df (#39350)

Contains the following:

* LUCENE-8635: Move terms dictionary off-heap for non-primary-key fields in `MMapDirectory`
* LUCENE-8292: `TermsEnum` is fully abstract
* LUCENE-8679: Return WITHIN in `EdgeTree#relateTriangle` only when polygon and triangle 
share one edge
* LUCENE-8676: Nori tokenizer deals correctly with large buffers
* LUCENE-8697: `GraphTokenStreamFiniteStrings` better handles side paths with gaps
* LUCENE-8664: Add `equals` and `hashCode` to `TotalHits`
* LUCENE-8660: `TopDocsCollector` returns accurate hit counts if the total equals the threshold
* LUCENE-8654: `Polygon2D#relateTriangle` fix for when the polygon is inside the triangle
* LUCENE-8645: `Intervals#fixField` can merge intervals from different fields
* LUCENE-8585: Create jump-tables for DocValues at index time

build.gradle

@@ -160,8 +160,8 @@ task verifyVersions {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = true
-final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
+boolean bwc_tests_enabled = false
+final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/39350" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

buildSrc/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 8.0.0
-lucene            = 8.0.0-snapshot-83f9835
+lucene            = 8.0.0-snapshot-ff9509a8df
 
 # optional dependencies
 spatial4j         = 0.7

modules/lang-expression/licenses/lucene-expressions-8.0.0-snapshot-83f9835.jar.sha1

@@ -1 +0,0 @@
-c84431b751d851f484f2799f6dcb9110113b1958

modules/lang-expression/licenses/lucene-expressions-8.0.0-snapshot-ff9509a8df.jar.sha1

@@ -0,0 +1 @@
+6b487ff516c053fae3fa270cd51964534f46b628

plugins/analysis-icu/licenses/lucene-analyzers-icu-8.0.0-snapshot-83f9835.jar.sha1

@@ -1 +0,0 @@
-ea440f53a9e858c2ed87927c63d57eb70e5af9ee

plugins/analysis-icu/licenses/lucene-analyzers-icu-8.0.0-snapshot-ff9509a8df.jar.sha1

@@ -0,0 +1 @@
+116fed6827a68b78b46a1eadfda1e27936b171a1

plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.0.0-snapshot-83f9835.jar.sha1

@@ -1 +0,0 @@
-b016fdfb8f1ac413d902cd4d244b295c9f01e610

plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.0.0-snapshot-ff9509a8df.jar.sha1

@@ -0,0 +1 @@
+c39659fa1684742370dfabf37641e41394ac4799

plugins/analysis-nori/licenses/lucene-analyzers-nori-8.0.0-snapshot-83f9835.jar.sha1

@@ -1 +0,0 @@
-395c8291751ffa7fbbb96faf578408a33a34ad1d

plugins/analysis-nori/licenses/lucene-analyzers-nori-8.0.0-snapshot-ff9509a8df.jar.sha1

@@ -0,0 +1 @@
+51e35640ceac2e4801b9be1263c7bea8637c7ad7

plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.0.0-snapshot-83f9835.jar.sha1

@@ -1 +0,0 @@
-6957c71604356a1fbf0e13794595e4ea42126dd7

plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.0.0-snapshot-ff9509a8df.jar.sha1

@@ -0,0 +1 @@
+bd8f2fe37a3e2f1c4d052071ceba618e6d16f119

plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.0.0-snapshot-83f9835.jar.sha1

@@ -1 +0,0 @@
-09f42235a8daca4ca8ea604314c2eb9de51b9e98

plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.0.0-snapshot-ff9509a8df.jar.sha1

@@ -0,0 +1 @@
+022de0054e9597d16f366bb83d6404d2006dcecf

plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.0.0-snapshot-83f9835.jar.sha1

@@ -1 +0,0 @@
-7cf7eeb685a2060e97f50551236cfcaf39990083

plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.0.0-snapshot-ff9509a8df.jar.sha1

@@ -0,0 +1 @@
+6cff98d0fdbb6d2d65ce9051005954f5b565986c

plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.0.0-snapshot-83f9835.jar.sha1

@@ -1 +0,0 @@
-ff163fb06ec3e47d501b290d8f69cca88a0341cc

plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.0.0-snapshot-ff9509a8df.jar.sha1

@@ -0,0 +1 @@
+438537046ef60d75f17358bf09bb339aa84191e5

server/licenses/lucene-analyzers-common-8.0.0-snapshot-83f9835.jar.sha1

@@ -1 +0,0 @@
-f4d286ed7940fc206331424e8ce577584208eba3

server/licenses/lucene-analyzers-common-8.0.0-snapshot-ff9509a8df.jar.sha1

@@ -0,0 +1 @@
+d96364f1b6a041f797e3d5f55b45284d8ab5cf95

server/licenses/lucene-backward-codecs-8.0.0-snapshot-83f9835.jar.sha1

@@ -1 +0,0 @@
-e5416da7370f5def9a79fb1cccb091b091b808a5

server/licenses/lucene-backward-codecs-8.0.0-snapshot-ff9509a8df.jar.sha1

@@ -0,0 +1 @@
+a39f5683e6e6ce17a191dd5fcbf90fcc43d25109

server/licenses/lucene-core-8.0.0-snapshot-83f9835.jar.sha1

@@ -1 +0,0 @@
-e3a95ff3cbd96e2c05b90932c20ca6374cdcdbe9

server/licenses/lucene-core-8.0.0-snapshot-ff9509a8df.jar.sha1

@@ -0,0 +1 @@
+283cbab4d58f18fd4eed169448bd0114e5d7c971

server/licenses/lucene-grouping-8.0.0-snapshot-83f9835.jar.sha1

@@ -1 +0,0 @@
-045dcae6368a436cec6e56f654a9c6b3a4656d17

server/licenses/lucene-grouping-8.0.0-snapshot-ff9509a8df.jar.sha1

@@ -0,0 +1 @@
+e63de6e4690f844b571edef03e4029a63bf2458b

server/licenses/lucene-highlighter-8.0.0-snapshot-83f9835.jar.sha1

@@ -1 +0,0 @@
-63b7164d24547cea36df3d81eedb9e8b424cf3c2

server/licenses/lucene-highlighter-8.0.0-snapshot-ff9509a8df.jar.sha1

@@ -0,0 +1 @@
+097cfc9ed8575c845c32f80745af11c04aaa6541

server/licenses/lucene-join-8.0.0-snapshot-83f9835.jar.sha1

@@ -1 +0,0 @@
-7b3358889c491237709e8aea4a39e816227f3d26

server/licenses/lucene-join-8.0.0-snapshot-ff9509a8df.jar.sha1

@@ -0,0 +1 @@
+a1b20c5408c9a1d2fec4604dc66b3d54bf91f84c

server/licenses/lucene-memory-8.0.0-snapshot-83f9835.jar.sha1

@@ -1 +0,0 @@
-5d9566e2b3be41c81fe42df1c57fff31fcdbc565

server/licenses/lucene-memory-8.0.0-snapshot-ff9509a8df.jar.sha1

@@ -0,0 +1 @@
+14bf9224dd2cd13d81ff90a43af713bda9c5b699

server/licenses/lucene-misc-8.0.0-snapshot-83f9835.jar.sha1

@@ -1 +0,0 @@
-45574e22c4b1569a2aecd5b2837388c7287892b5

server/licenses/lucene-misc-8.0.0-snapshot-ff9509a8df.jar.sha1

@@ -0,0 +1 @@
+79be1830102a764d79da6c81d40f5f6bf85ad032

server/licenses/lucene-queries-8.0.0-snapshot-83f9835.jar.sha1

@@ -1 +0,0 @@
-67405660330db8c09e0994a615cf3ab95d7bc151

server/licenses/lucene-queries-8.0.0-snapshot-ff9509a8df.jar.sha1

@@ -0,0 +1 @@
+3f7281351f03ac34c8a950e667e826053b82f31e

server/licenses/lucene-queryparser-8.0.0-snapshot-83f9835.jar.sha1

@@ -1 +0,0 @@
-c8de64bf2c4f09766d4cd62e8dd403016665f37c

server/licenses/lucene-queryparser-8.0.0-snapshot-ff9509a8df.jar.sha1

@@ -0,0 +1 @@
+039229d20cb500de435b458998aa35370c616f69

server/licenses/lucene-sandbox-8.0.0-snapshot-83f9835.jar.sha1

@@ -1 +0,0 @@
-fc080b45b881a78a23743df9a7388fd3dbbb9d66

server/licenses/lucene-sandbox-8.0.0-snapshot-ff9509a8df.jar.sha1

@@ -0,0 +1 @@
+00cee2f9ff5fcaa3ac3838f2d8b0c8b59c55e4a6

server/licenses/lucene-spatial-8.0.0-snapshot-83f9835.jar.sha1

@@ -1 +0,0 @@
-6e0e12b9882005b87ef39325e1fc8539c8caff31

server/licenses/lucene-spatial-8.0.0-snapshot-ff9509a8df.jar.sha1

@@ -0,0 +1 @@
+4246d1cc124eb7b2d18ccee1db96b5999ae45b95

server/licenses/lucene-spatial-extras-8.0.0-snapshot-83f9835.jar.sha1

@@ -1 +0,0 @@
-7ed9c9a03e1a15840237d09ca4da7aadfce1c780

server/licenses/lucene-spatial-extras-8.0.0-snapshot-ff9509a8df.jar.sha1

@@ -0,0 +1 @@
+caa0c22ce9a985f4a7a76c315ac8841d37083c2b

server/licenses/lucene-spatial3d-8.0.0-snapshot-83f9835.jar.sha1

@@ -1 +0,0 @@
-fdf0efdeb9a73a6b0f8349df21049ecbc73955d7

server/licenses/lucene-spatial3d-8.0.0-snapshot-ff9509a8df.jar.sha1

@@ -0,0 +1 @@
+7d01e71c2e2e9fd5a2ad1f397bf75de12b860fab

server/licenses/lucene-suggest-8.0.0-snapshot-83f9835.jar.sha1

@@ -1 +0,0 @@
-c69399183d5f9f85f3f8130452d0bed62fd92440

server/licenses/lucene-suggest-8.0.0-snapshot-ff9509a8df.jar.sha1

@@ -0,0 +1 @@
+fd72490e235dffbf455886d4a4339dd29f4ccd5b

server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsFields.java

@@ -21,6 +21,7 @@ package org.elasticsearch.action.termvectors;
 
 import com.carrotsearch.hppc.ObjectLongHashMap;
 import com.carrotsearch.hppc.cursors.ObjectLongCursor;
+import org.apache.lucene.index.BaseTermsEnum;
 import org.apache.lucene.index.Fields;
 import org.apache.lucene.index.ImpactsEnum;
 import org.apache.lucene.index.PostingsEnum;
@@ -229,7 +230,7 @@ public final class TermVectorsFields extends Fields {
             // reset before asking for an iterator
             reset();
             // convert bytes ref for the terms to actual data
-            return new TermsEnum() {
+            return new BaseTermsEnum() {
                 int currentTerm = 0;
                 int freq = 0;
                 int docFreq = -1;

server/src/main/java/org/elasticsearch/common/lucene/index/FilterableTermsEnum.java

@@ -23,6 +23,7 @@ import org.apache.lucene.index.ImpactsEnum;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.PostingsEnum;
+import org.apache.lucene.index.TermState;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.search.DocIdSetIterator;
@@ -32,6 +33,7 @@ import org.apache.lucene.search.Query;
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.Weight;
+import org.apache.lucene.util.AttributeSource;
 import org.apache.lucene.util.BitSet;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
@@ -126,6 +128,11 @@ public class FilterableTermsEnum extends TermsEnum {
         return current;
     }
 
+    @Override
+    public AttributeSource attributes() {
+        throw new UnsupportedOperationException(UNSUPPORTED_MESSAGE);
+    }
+
     @Override
     public boolean seekExact(BytesRef text) throws IOException {
         int docFreq = 0;
@@ -195,6 +202,16 @@ public class FilterableTermsEnum extends TermsEnum {
         throw new UnsupportedOperationException(UNSUPPORTED_MESSAGE);
     }
 
+    @Override
+    public void seekExact(BytesRef term, TermState state) throws IOException {
+        throw new UnsupportedOperationException(UNSUPPORTED_MESSAGE);
+    }
+
+    @Override
+    public TermState termState() throws IOException {
+        throw new UnsupportedOperationException(UNSUPPORTED_MESSAGE);
+    }
+
     @Override
     public SeekStatus seekCeil(BytesRef text) throws IOException {
         throw new UnsupportedOperationException(UNSUPPORTED_MESSAGE);

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java

@@ -19,6 +19,7 @@ import org.apache.lucene.index.SortedDocValues;
 import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.index.StoredFieldVisitor;
+import org.apache.lucene.index.TermState;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.util.BytesRef;
@@ -441,6 +442,14 @@ public final class FieldSubsetReader extends FilterLeafReader {
             return accept(term) && in.seekExact(term);
         }
 
+        @Override
+        public void seekExact(BytesRef term, TermState state) throws IOException {
+            if (accept(term) == false) {
+                throw new IllegalStateException("Tried to seek using a TermState from a different reader!");
+            }
+            in.seekExact(term, state);
+        }
+
         @Override
         public SeekStatus seekCeil(BytesRef term) throws IOException {
             SeekStatus status = in.seekCeil(term);

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java

@@ -23,6 +23,7 @@ import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.Fields;
+import org.apache.lucene.index.FilterDirectoryReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.LeafReader;
@@ -35,12 +36,12 @@ import org.apache.lucene.index.SortedDocValues;
 import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermState;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.index.TermsEnum.SeekStatus;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.core.internal.io.IOUtils;
 import org.apache.lucene.util.TestUtil;
 import org.apache.lucene.util.automaton.Automata;
 import org.apache.lucene.util.automaton.Automaton;
@@ -55,6 +56,7 @@ import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.test.ESTestCase;
@@ -768,6 +770,18 @@ public class FieldSubsetReaderTests extends ESTestCase {
         assertTrue(termsEnum.seekExact(new BytesRef("fieldA")));
         assertFalse(termsEnum.seekExact(new BytesRef("fieldB")));
 
+        // seekExact with TermState
+        // first, collect TermState from underlying reader
+        LeafReader unwrappedReader = FilterDirectoryReader.unwrap(ir).leaves().get(0).reader();
+        Terms unwrappedTerms = unwrappedReader.terms(FieldNamesFieldMapper.NAME);
+        TermsEnum unwrappedTE = unwrappedTerms.iterator();
+        assertTrue(unwrappedTE.seekExact(new BytesRef("fieldB")));
+        TermState termState = unwrappedTE.termState();
+
+        // now try and seekExact with it
+        TermsEnum badEnum = terms.iterator();
+        expectThrows(IllegalStateException.class, () -> badEnum.seekExact(new BytesRef("fieldB"), termState));
+
         // seekCeil
         termsEnum = terms.iterator();
         assertEquals(SeekStatus.FOUND, termsEnum.seekCeil(new BytesRef("fieldA")));

x-pack/plugin/sql/sql-action/licenses/lucene-core-8.0.0-snapshot-83f9835.jar.sha1

@@ -1 +0,0 @@
-e3a95ff3cbd96e2c05b90932c20ca6374cdcdbe9

x-pack/plugin/sql/sql-action/licenses/lucene-core-8.0.0-snapshot-ff9509a8df.jar.sha1

@@ -0,0 +1 @@
+283cbab4d58f18fd4eed169448bd0114e5d7c971