Support 7x segments as archive in 8x / 9x (#119503)

Added BWCLucene8*Codecs wrapper classes for the  lucene8* equivalents. A BWC wrapper is initialized for archive indices and provides read-only capabilities for an index.

docs/changelog/119503.yaml

@@ -0,0 +1,6 @@
+pr: 119503
+summary: Support indices created in ESv6 and updated in ESV7 using different LuceneCodecs as archive in current version.
+area: Search
+type: bug
+issues:
+ - 117042

x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java

@@ -17,6 +17,7 @@ import org.apache.lucene.codecs.NormsProducer;
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.codecs.SegmentInfoFormat;
 import org.apache.lucene.codecs.TermVectorsFormat;
+import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.Fields;
@@ -26,6 +27,13 @@ import org.apache.lucene.index.SegmentWriteState;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
+import org.apache.lucene.util.Version;
+import org.elasticsearch.core.UpdateForV10;
+import org.elasticsearch.xpack.lucene.bwc.codecs.lucene70.BWCLucene70Codec;
+import org.elasticsearch.xpack.lucene.bwc.codecs.lucene80.BWCLucene80Codec;
+import org.elasticsearch.xpack.lucene.bwc.codecs.lucene84.BWCLucene84Codec;
+import org.elasticsearch.xpack.lucene.bwc.codecs.lucene86.BWCLucene86Codec;
+import org.elasticsearch.xpack.lucene.bwc.codecs.lucene87.BWCLucene87Codec;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -37,55 +45,122 @@ import java.util.List;
  */
 public abstract class BWCCodec extends Codec {
 
+    private final FieldInfosFormat fieldInfosFormat;
+    private final SegmentInfoFormat segmentInfosFormat;
+    private final PostingsFormat postingsFormat;
+
     protected BWCCodec(String name) {
         super(name);
-    }
 
-    @Override
-    public NormsFormat normsFormat() {
-        throw new UnsupportedOperationException();
-    }
+        this.fieldInfosFormat = new FieldInfosFormat() {
+            final FieldInfosFormat wrappedFormat = originalFieldInfosFormat();
 
-    @Override
-    public TermVectorsFormat termVectorsFormat() {
-        throw new UnsupportedOperationException();
-    }
+            @Override
+            public FieldInfos read(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, IOContext iocontext)
+                throws IOException {
+                return filterFields(wrappedFormat.read(directory, segmentInfo, segmentSuffix, iocontext));
+            }
 
-    @Override
-    public KnnVectorsFormat knnVectorsFormat() {
-        throw new UnsupportedOperationException();
-    }
+            @Override
+            public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context)
+                throws IOException {
+                wrappedFormat.write(directory, segmentInfo, segmentSuffix, infos, context);
+            }
+        };
+
+        this.segmentInfosFormat = new SegmentInfoFormat() {
+            final SegmentInfoFormat wrappedFormat = originalSegmentInfoFormat();
 
-    protected static SegmentInfoFormat wrap(SegmentInfoFormat wrapped) {
-        return new SegmentInfoFormat() {
             @Override
             public SegmentInfo read(Directory directory, String segmentName, byte[] segmentID, IOContext context) throws IOException {
-                return wrap(wrapped.read(directory, segmentName, segmentID, context));
+                return wrap(wrappedFormat.read(directory, segmentName, segmentID, context));
             }
 
             @Override
             public void write(Directory dir, SegmentInfo info, IOContext ioContext) throws IOException {
-                wrapped.write(dir, info, ioContext);
+                wrappedFormat.write(dir, info, ioContext);
             }
         };
-    }
 
-    protected static FieldInfosFormat wrap(FieldInfosFormat wrapped) {
-        return new FieldInfosFormat() {
+        this.postingsFormat = new PerFieldPostingsFormat() {
             @Override
-            public FieldInfos read(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, IOContext iocontext)
-                throws IOException {
-                return filterFields(wrapped.read(directory, segmentInfo, segmentSuffix, iocontext));
-            }
-
-            @Override
-            public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context)
-                throws IOException {
-                wrapped.write(directory, segmentInfo, segmentSuffix, infos, context);
+            public PostingsFormat getPostingsFormatForField(String field) {
+                throw new UnsupportedOperationException("Old codecs can't be used for writing");
             }
         };
     }
 
+    @Override
+    public final FieldInfosFormat fieldInfosFormat() {
+        return fieldInfosFormat;
+    }
+
+    @Override
+    public final SegmentInfoFormat segmentInfoFormat() {
+        return segmentInfosFormat;
+    }
+
+    @Override
+    public PostingsFormat postingsFormat() {
+        return postingsFormat;
+    }
+
+    /**
+     * This method is not supported for archive indices and older codecs and will always throw an {@link UnsupportedOperationException}.
+     * This method is never called in practice, as we rewrite field infos to override the info about which features are present in
+     * the index. Even if norms are present, field info lies about it.
+     *
+     * @return nothing, as this method always throws an exception
+     * @throws UnsupportedOperationException always thrown to indicate that this method is not supported
+     */
+    @Override
+    public final NormsFormat normsFormat() {
+        throw new UnsupportedOperationException();
+    }
+
+    /**
+     * This method is not supported for archive indices and older codecs and will always throw an {@link UnsupportedOperationException}.
+     * This method is never called in practice, as we rewrite field infos to override the info about which features are present in
+     * the index. Even if term vectors are present, field info lies about it.
+     *
+     * @return nothing, as this method always throws an exception
+     * @throws UnsupportedOperationException always thrown to indicate that this method is not supported
+     */
+    @Override
+    public final TermVectorsFormat termVectorsFormat() {
+        throw new UnsupportedOperationException();
+    }
+
+    /**
+     * This method is not supported for archive indices and older codecs and will always throw an {@link UnsupportedOperationException}.
+     * The knn vectors can't be present because it is not supported yet in any of the lucene versions that we support for archive indices.
+     *
+     * @return nothing, as this method always throws an exception
+     * @throws UnsupportedOperationException always thrown to indicate that this method is not supported
+     */
+    @Override
+    public final KnnVectorsFormat knnVectorsFormat() {
+        throw new UnsupportedOperationException();
+    }
+
+    /**
+     * Returns the original {@link SegmentInfoFormat} used by this codec.
+     * This method should be implemented by subclasses to provide the specific
+     * {@link SegmentInfoFormat} that this codec is intended to use.
+     *
+     * @return the original {@link SegmentInfoFormat} used by this codec
+     */
+    protected abstract SegmentInfoFormat originalSegmentInfoFormat();
+
+    /**
+     * Returns the original {@link FieldInfosFormat} used by this codec.
+     * This method should be implemented by subclasses to provide the specific
+     * {@link FieldInfosFormat} that this codec is intended to use.
+     *
+     * @return the original {@link FieldInfosFormat} used by this codec
+     */
+    protected abstract FieldInfosFormat originalFieldInfosFormat();
+
     // mark all fields as no term vectors, no norms, no payloads, and no vectors.
     private static FieldInfos filterFields(FieldInfos fieldInfos) {
         List<FieldInfo> fieldInfoCopy = new ArrayList<>(fieldInfos.size());
@@ -118,13 +193,14 @@ public abstract class BWCCodec extends Codec {
     }
 
     public static SegmentInfo wrap(SegmentInfo segmentInfo) {
-        final Codec codec = segmentInfo.getCodec();
+        Codec codec = getBackwardCompatibleCodec(segmentInfo.getCodec());
+
         final SegmentInfo segmentInfo1 = new SegmentInfo(
             segmentInfo.dir,
             // Use Version.LATEST instead of original version, otherwise SegmentCommitInfo will bark when processing (N-1 limitation)
             // TODO: perhaps store the original version information in attributes so that we can retrieve it later when needed?
-            org.apache.lucene.util.Version.LATEST,
-            org.apache.lucene.util.Version.LATEST,
+            Version.LATEST,
+            Version.LATEST,
             segmentInfo.name,
             segmentInfo.maxDoc(),
             segmentInfo.getUseCompoundFile(),
@@ -139,6 +215,29 @@ public abstract class BWCCodec extends Codec {
         return segmentInfo1;
     }
 
+    /**
+     * Returns a backward-compatible codec for the given codec. If the codec is one of the known Lucene 8.x codecs,
+     * it returns a corresponding read-only backward-compatible codec. Otherwise, it returns the original codec.
+     * Lucene 8.x codecs are still shipped with the current version of Lucene.
+     * Earlier codecs we are providing directly they will also be read-only backward-compatible, but they don't require the renaming.
+     *
+     * This switch is only for indices created in ES 6.x, later written into in ES 7.x (Lucene 8.x). Indices created
+     * in ES 7.x can be read directly by ES if marked read-only, without going through archive indices.
+     */
+    @UpdateForV10(owner = UpdateForV10.Owner.SEARCH_FOUNDATIONS)
+    private static Codec getBackwardCompatibleCodec(Codec codec) {
+        if (codec == null) return null;
+
+        return switch (codec.getClass().getSimpleName()) {
+            case "Lucene70Codec" -> new BWCLucene70Codec();
+            case "Lucene80Codec" -> new BWCLucene80Codec();
+            case "Lucene84Codec" -> new BWCLucene84Codec();
+            case "Lucene86Codec" -> new BWCLucene86Codec();
+            case "Lucene87Codec" -> new BWCLucene87Codec();
+            default -> codec;
+        };
+    }
+
     /**
      * In-memory postings format that shows no postings available.
      */

x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60Codec.java

@@ -44,8 +44,7 @@ import org.elasticsearch.xpack.lucene.bwc.codecs.lucene54.Lucene54DocValuesForma
  */
 @Deprecated
 public class Lucene60Codec extends BWCCodec {
-    private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat());
-    private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene50SegmentInfoFormat());
+
     private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat();
     private final CompoundFormat compoundFormat = new Lucene50CompoundFormat();
     private final StoredFieldsFormat storedFieldsFormat;
@@ -68,18 +67,18 @@ public class Lucene60Codec extends BWCCodec {
     }
 
     @Override
-    public final StoredFieldsFormat storedFieldsFormat() {
-        return storedFieldsFormat;
+    protected FieldInfosFormat originalFieldInfosFormat() {
+        return new Lucene60FieldInfosFormat();
     }
 
     @Override
-    public final FieldInfosFormat fieldInfosFormat() {
-        return fieldInfosFormat;
+    protected SegmentInfoFormat originalSegmentInfoFormat() {
+        return new Lucene50SegmentInfoFormat();
     }
 
     @Override
-    public SegmentInfoFormat segmentInfoFormat() {
-        return segmentInfosFormat;
+    public final StoredFieldsFormat storedFieldsFormat() {
+        return storedFieldsFormat;
     }
 
     @Override

x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60MetadataOnlyPointsFormat.java

@@ -28,6 +28,7 @@ import org.apache.lucene.index.SegmentWriteState;
 import java.io.IOException;
 
 /**
+ * This is a fork of {@link org.apache.lucene.backward_codecs.lucene60.Lucene60PointsFormat}
  * Allows reading metadata only from Lucene 6.0 point format
  **/
 public class Lucene60MetadataOnlyPointsFormat extends PointsFormat {

x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60MetadataOnlyPointsReader.java

@@ -34,7 +34,10 @@ import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 
-/** Reads the metadata of point values previously written with Lucene60PointsWriter */
+/**
+ * This is a fork of {@link org.apache.lucene.backward_codecs.lucene60.Lucene60PointsReader}
+ * Reads the metadata of point values previously written with Lucene60PointsWriter
+ */
 public final class Lucene60MetadataOnlyPointsReader extends PointsReader {
     final IndexInput dataIn;
     final SegmentReadState readState;
@@ -105,7 +108,7 @@ public final class Lucene60MetadataOnlyPointsReader extends PointsReader {
                 int fieldNumber = ent.getKey();
                 long fp = ent.getValue();
                 dataIn.seek(fp);
-                PointValues reader = new MetadataOnlyBKDReader(dataIn);
+                PointValues reader = new MetadataOnlyBKDReader(dataIn, false);
                 readers.put(fieldNumber, reader);
             }
 

x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/MetadataOnlyBKDReader.java

@@ -47,7 +47,7 @@ public class MetadataOnlyBKDReader extends PointValues {
     final int docCount;
     final int version;
 
-    public MetadataOnlyBKDReader(IndexInput metaIn) throws IOException {
+    public MetadataOnlyBKDReader(IndexInput metaIn, boolean isVersionPost86) throws IOException {
         version = CodecUtil.checkHeader(metaIn, "BKD", VERSION_START, VERSION_CURRENT);
         final int numDims = metaIn.readVInt();
         final int numIndexDims;
@@ -85,6 +85,15 @@ public class MetadataOnlyBKDReader extends PointValues {
 
         pointCount = metaIn.readVLong();
         docCount = metaIn.readVInt();
+
+        // This code has been introduced to process IndexInput created with Lucene86Codec+. This is not necessary
+        // in the read-only version for older formats.
+        if (isVersionPost86) {
+            metaIn.readVInt();
+            metaIn.readLong();
+            // The following fields are not used in this class, but we need to read them to advance the pointer
+            metaIn.readLong();
+        }
     }
 
     @Override

x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene62/Lucene62Codec.java

@@ -44,8 +44,7 @@ import org.elasticsearch.xpack.lucene.bwc.codecs.lucene60.Lucene60MetadataOnlyPo
  */
 @Deprecated
 public class Lucene62Codec extends BWCCodec {
-    private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat());
-    private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene62SegmentInfoFormat());
+
     private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat();
     private final CompoundFormat compoundFormat = new Lucene50CompoundFormat();
     private final StoredFieldsFormat storedFieldsFormat;
@@ -68,18 +67,18 @@ public class Lucene62Codec extends BWCCodec {
     }
 
     @Override
-    public final StoredFieldsFormat storedFieldsFormat() {
-        return storedFieldsFormat;
+    protected FieldInfosFormat originalFieldInfosFormat() {
+        return new Lucene60FieldInfosFormat();
     }
 
     @Override
-    public final FieldInfosFormat fieldInfosFormat() {
-        return fieldInfosFormat;
+    protected SegmentInfoFormat originalSegmentInfoFormat() {
+        return new Lucene62SegmentInfoFormat();
     }
 
     @Override
-    public SegmentInfoFormat segmentInfoFormat() {
-        return segmentInfosFormat;
+    public final StoredFieldsFormat storedFieldsFormat() {
+        return storedFieldsFormat;
     }
 
     @Override

x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java

@@ -17,11 +17,9 @@ import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.FieldInfosFormat;
 import org.apache.lucene.codecs.LiveDocsFormat;
 import org.apache.lucene.codecs.PointsFormat;
-import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.codecs.SegmentInfoFormat;
 import org.apache.lucene.codecs.StoredFieldsFormat;
 import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
-import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
 import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec;
 import org.elasticsearch.xpack.lucene.bwc.codecs.lucene60.Lucene60MetadataOnlyPointsFormat;
 
@@ -33,8 +31,6 @@ import org.elasticsearch.xpack.lucene.bwc.codecs.lucene60.Lucene60MetadataOnlyPo
  */
 public class BWCLucene70Codec extends BWCCodec {
 
-    private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat());
-    private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene70SegmentInfoFormat());
     private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat();
     private final CompoundFormat compoundFormat = new Lucene50CompoundFormat();
     private final StoredFieldsFormat storedFieldsFormat;
@@ -45,12 +41,7 @@ public class BWCLucene70Codec extends BWCCodec {
             return defaultDVFormat;
         }
     };
-    private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() {
-        @Override
-        public PostingsFormat getPostingsFormatForField(String field) {
-            throw new IllegalStateException("This codec should only be used for reading, not writing");
-        }
-    };
+    private final PointsFormat pointsFormat = new Lucene60MetadataOnlyPointsFormat();
 
     // Needed for SPI loading
     @SuppressWarnings("unused")
@@ -64,13 +55,13 @@ public class BWCLucene70Codec extends BWCCodec {
     }
 
     @Override
-    public FieldInfosFormat fieldInfosFormat() {
-        return fieldInfosFormat;
+    protected FieldInfosFormat originalFieldInfosFormat() {
+        return new Lucene60FieldInfosFormat();
     }
 
     @Override
-    public SegmentInfoFormat segmentInfoFormat() {
-        return segmentInfosFormat;
+    protected SegmentInfoFormat originalSegmentInfoFormat() {
+        return new Lucene70SegmentInfoFormat();
     }
 
     @Override
@@ -93,13 +84,8 @@ public class BWCLucene70Codec extends BWCCodec {
         return docValuesFormat;
     }
 
-    @Override
-    public PostingsFormat postingsFormat() {
-        return postingsFormat;
-    }
-
     @Override
     public PointsFormat pointsFormat() {
-        return new Lucene60MetadataOnlyPointsFormat();
+        return pointsFormat;
     }
 }

x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene80/BWCLucene80Codec.java

@@ -0,0 +1,89 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.lucene.bwc.codecs.lucene80;
+
+import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat;
+import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat;
+import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat;
+import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat;
+import org.apache.lucene.backward_codecs.lucene70.Lucene70SegmentInfoFormat;
+import org.apache.lucene.backward_codecs.lucene80.Lucene80DocValuesFormat;
+import org.apache.lucene.codecs.CompoundFormat;
+import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.FieldInfosFormat;
+import org.apache.lucene.codecs.LiveDocsFormat;
+import org.apache.lucene.codecs.PointsFormat;
+import org.apache.lucene.codecs.SegmentInfoFormat;
+import org.apache.lucene.codecs.StoredFieldsFormat;
+import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
+import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec;
+import org.elasticsearch.xpack.lucene.bwc.codecs.lucene60.Lucene60MetadataOnlyPointsFormat;
+
+/**
+ * This is a fork of {@link org.apache.lucene.backward_codecs.lucene80.Lucene80Codec}
+ * Implements the Lucene 8.0 index format. Loaded via SPI for indices created/written with Lucene 8.0.0-8.3.0
+ * (Elasticsearch [7.0.0-7.5.2]), mounted as archive indices in Elasticsearch 8.x / 9.x.
+ */
+public class BWCLucene80Codec extends BWCCodec {
+
+    private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat();
+    private final CompoundFormat compoundFormat = new Lucene50CompoundFormat();
+
+    private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() {
+        @Override
+        public DocValuesFormat getDocValuesFormatForField(String field) {
+            return defaultDVFormat;
+        }
+    };
+    private final DocValuesFormat defaultDVFormat = new Lucene80DocValuesFormat();
+
+    private final StoredFieldsFormat storedFieldsFormat;
+    private final PointsFormat pointsFormat = new Lucene60MetadataOnlyPointsFormat();
+
+    // Needed for SPI loading
+    @SuppressWarnings("unused")
+    public BWCLucene80Codec() {
+        super("BWCLucene80Codec");
+        this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Lucene50StoredFieldsFormat.Mode.BEST_SPEED);
+    }
+
+    @Override
+    protected FieldInfosFormat originalFieldInfosFormat() {
+        return new Lucene60FieldInfosFormat();
+    }
+
+    @Override
+    protected SegmentInfoFormat originalSegmentInfoFormat() {
+        return new Lucene70SegmentInfoFormat();
+    }
+
+    @Override
+    public final StoredFieldsFormat storedFieldsFormat() {
+        return storedFieldsFormat;
+    }
+
+    @Override
+    public final LiveDocsFormat liveDocsFormat() {
+        return liveDocsFormat;
+    }
+
+    @Override
+    public final CompoundFormat compoundFormat() {
+        return compoundFormat;
+    }
+
+    @Override
+    public final PointsFormat pointsFormat() {
+        return pointsFormat;
+    }
+
+    @Override
+    public final DocValuesFormat docValuesFormat() {
+        return docValuesFormat;
+    }
+}

x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene84/BWCLucene84Codec.java

@@ -0,0 +1,90 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.lucene.bwc.codecs.lucene84;
+
+import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat;
+import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat;
+import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat;
+import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat;
+import org.apache.lucene.backward_codecs.lucene70.Lucene70SegmentInfoFormat;
+import org.apache.lucene.backward_codecs.lucene80.Lucene80DocValuesFormat;
+import org.apache.lucene.codecs.CompoundFormat;
+import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.FieldInfosFormat;
+import org.apache.lucene.codecs.LiveDocsFormat;
+import org.apache.lucene.codecs.PointsFormat;
+import org.apache.lucene.codecs.SegmentInfoFormat;
+import org.apache.lucene.codecs.StoredFieldsFormat;
+import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
+import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec;
+import org.elasticsearch.xpack.lucene.bwc.codecs.lucene60.Lucene60MetadataOnlyPointsFormat;
+
+/**
+ * This is a fork of {@link org.apache.lucene.backward_codecs.lucene84.Lucene84Codec}
+ * Implements the Lucene 8.4 index format. Loaded via SPI for indices created/written with Lucene 8.4.0-8.5.1
+ * (Elasticsearch [7.6.0-7.8.1]), mounted as archive indices in Elasticsearch 8.x / 9.x.
+ */
+public class BWCLucene84Codec extends BWCCodec {
+
+    private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat();
+    private final CompoundFormat compoundFormat = new Lucene50CompoundFormat();
+    private final DocValuesFormat defaultDVFormat;
+
+    private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() {
+        @Override
+        public DocValuesFormat getDocValuesFormatForField(String field) {
+            return defaultDVFormat;
+        }
+    };
+
+    private final StoredFieldsFormat storedFieldsFormat;
+    private final PointsFormat pointsFormat = new Lucene60MetadataOnlyPointsFormat();
+
+    // Needed for SPI loading
+    @SuppressWarnings("unused")
+    public BWCLucene84Codec() {
+        super("BWCLucene84Codec");
+        this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Lucene50StoredFieldsFormat.Mode.BEST_SPEED);
+        this.defaultDVFormat = new Lucene80DocValuesFormat();
+    }
+
+    @Override
+    protected FieldInfosFormat originalFieldInfosFormat() {
+        return new Lucene60FieldInfosFormat();
+    }
+
+    @Override
+    protected SegmentInfoFormat originalSegmentInfoFormat() {
+        return new Lucene70SegmentInfoFormat();
+    }
+
+    @Override
+    public StoredFieldsFormat storedFieldsFormat() {
+        return storedFieldsFormat;
+    }
+
+    @Override
+    public final LiveDocsFormat liveDocsFormat() {
+        return liveDocsFormat;
+    }
+
+    @Override
+    public CompoundFormat compoundFormat() {
+        return compoundFormat;
+    }
+
+    @Override
+    public PointsFormat pointsFormat() {
+        return pointsFormat;
+    }
+
+    @Override
+    public final DocValuesFormat docValuesFormat() {
+        return docValuesFormat;
+    }
+}

x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/BWCLucene86Codec.java

@@ -0,0 +1,89 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.lucene.bwc.codecs.lucene86;
+
+import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat;
+import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat;
+import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat;
+import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat;
+import org.apache.lucene.backward_codecs.lucene80.Lucene80DocValuesFormat;
+import org.apache.lucene.backward_codecs.lucene86.Lucene86SegmentInfoFormat;
+import org.apache.lucene.codecs.CompoundFormat;
+import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.FieldInfosFormat;
+import org.apache.lucene.codecs.LiveDocsFormat;
+import org.apache.lucene.codecs.PointsFormat;
+import org.apache.lucene.codecs.SegmentInfoFormat;
+import org.apache.lucene.codecs.StoredFieldsFormat;
+import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
+import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec;
+
+/**
+ * This is a fork of {@link org.apache.lucene.backward_codecs.lucene86.Lucene86Codec}
+ * Implements the Lucene 8.6 index format. Loaded via SPI for indices created/written with Lucene 8.6.0-8.6.2
+ * (Elasticsearch [7.9.0-7.9.3]), mounted as archive indices in Elasticsearch 8.x / 9.x.
+ */
+public class BWCLucene86Codec extends BWCCodec {
+
+    private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat();
+    private final CompoundFormat compoundFormat = new Lucene50CompoundFormat();
+    private final PointsFormat pointsFormat = new Lucene86MetadataOnlyPointsFormat();
+    private final DocValuesFormat defaultDVFormat;
+
+    private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() {
+        @Override
+        public DocValuesFormat getDocValuesFormatForField(String field) {
+            return defaultDVFormat;
+        }
+    };
+
+    private final StoredFieldsFormat storedFieldsFormat;
+
+    // Needed for SPI loading
+    @SuppressWarnings("unused")
+    public BWCLucene86Codec() {
+        super("BWCLucene86Codec");
+        this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Lucene50StoredFieldsFormat.Mode.BEST_SPEED);
+        this.defaultDVFormat = new Lucene80DocValuesFormat();
+    }
+
+    @Override
+    protected FieldInfosFormat originalFieldInfosFormat() {
+        return new Lucene60FieldInfosFormat();
+    }
+
+    @Override
+    protected SegmentInfoFormat originalSegmentInfoFormat() {
+        return new Lucene86SegmentInfoFormat();
+    }
+
+    @Override
+    public StoredFieldsFormat storedFieldsFormat() {
+        return storedFieldsFormat;
+    }
+
+    @Override
+    public final LiveDocsFormat liveDocsFormat() {
+        return liveDocsFormat;
+    }
+
+    @Override
+    public CompoundFormat compoundFormat() {
+        return compoundFormat;
+    }
+
+    @Override
+    public PointsFormat pointsFormat() {
+        return pointsFormat;
+    }
+
+    @Override
+    public final DocValuesFormat docValuesFormat() {
+        return docValuesFormat;
+    }
+}

x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsFormat.java

@@ -0,0 +1,56 @@
+/*
+ * @notice
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Modifications copyright (C) 2021 Elasticsearch B.V.
+ */
+package org.elasticsearch.xpack.lucene.bwc.codecs.lucene86;
+
+import org.apache.lucene.codecs.PointsFormat;
+import org.apache.lucene.codecs.PointsReader;
+import org.apache.lucene.codecs.PointsWriter;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.SegmentWriteState;
+
+import java.io.IOException;
+
+/**
+ * This is a fork of {@link org.apache.lucene.backward_codecs.lucene86.Lucene86PointsFormat}
+ * Allows reading metadata only from Lucene 8.6 point format
+ **/
+public class Lucene86MetadataOnlyPointsFormat extends PointsFormat {
+
+    static final String META_CODEC_NAME = "Lucene86PointsFormatMeta";
+
+    /** Filename extension for the meta per field */
+    public static final String META_EXTENSION = "kdm";
+
+    static final int VERSION_START = 0;
+    static final int VERSION_CURRENT = VERSION_START;
+
+    /** Sole constructor */
+    public Lucene86MetadataOnlyPointsFormat() {}
+
+    @Override
+    public PointsWriter fieldsWriter(SegmentWriteState state) {
+        throw new UnsupportedOperationException("Old codecs may only be used for reading");
+    }
+
+    @Override
+    public PointsReader fieldsReader(SegmentReadState state) throws IOException {
+        return new Lucene86MetadataOnlyPointsReader(state);
+    }
+}

x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene86/Lucene86MetadataOnlyPointsReader.java

@@ -0,0 +1,121 @@
+/*
+ * @notice
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Modifications copyright (C) 2021 Elasticsearch B.V.
+ */
+package org.elasticsearch.xpack.lucene.bwc.codecs.lucene86;
+
+import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.codecs.PointsReader;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.PointValues;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.store.ChecksumIndexInput;
+import org.elasticsearch.core.IOUtils;
+import org.elasticsearch.xpack.lucene.bwc.codecs.lucene60.MetadataOnlyBKDReader;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ *  This is a fork of {@link org.apache.lucene.backward_codecs.lucene86.Lucene86PointsReader}
+ *  Reads the metadata of point values previously written with Lucene86PointsWriter
+ */
+public final class Lucene86MetadataOnlyPointsReader extends PointsReader {
+    final SegmentReadState readState;
+    final Map<Integer, PointValues> readers = new HashMap<>();
+
+    public Lucene86MetadataOnlyPointsReader(SegmentReadState readState) throws IOException {
+        this.readState = readState;
+
+        String metaFileName = IndexFileNames.segmentFileName(
+            readState.segmentInfo.name,
+            readState.segmentSuffix,
+            Lucene86MetadataOnlyPointsFormat.META_EXTENSION
+        );
+
+        boolean success = false;
+        try {
+            try (
+                ChecksumIndexInput metaIn = EndiannessReverserUtil.openChecksumInput(readState.directory, metaFileName, readState.context)
+            ) {
+                Throwable priorE = null;
+                try {
+                    CodecUtil.checkIndexHeader(
+                        metaIn,
+                        Lucene86MetadataOnlyPointsFormat.META_CODEC_NAME,
+                        Lucene86MetadataOnlyPointsFormat.VERSION_START,
+                        Lucene86MetadataOnlyPointsFormat.VERSION_CURRENT,
+                        readState.segmentInfo.getId(),
+                        readState.segmentSuffix
+                    );
+
+                    while (true) {
+                        int fieldNumber = metaIn.readInt();
+                        if (fieldNumber == -1) {
+                            break;
+                        } else if (fieldNumber < 0) {
+                            throw new CorruptIndexException("Illegal field number: " + fieldNumber, metaIn);
+                        }
+                        PointValues reader = new MetadataOnlyBKDReader(metaIn, true);
+                        readers.put(fieldNumber, reader);
+                    }
+                    metaIn.readLong();
+                    metaIn.readLong();
+                } catch (Throwable t) {
+                    priorE = t;
+                } finally {
+                    CodecUtil.checkFooter(metaIn, priorE);
+                }
+            }
+
+            success = true;
+        } finally {
+            if (success == false) {
+                IOUtils.closeWhileHandlingException(this);
+            }
+        }
+    }
+
+    @Override
+    public PointValues getValues(String fieldName) {
+        FieldInfo fieldInfo = readState.fieldInfos.fieldInfo(fieldName);
+        if (fieldInfo == null) {
+            throw new IllegalArgumentException("field=\"" + fieldName + "\" is unrecognized");
+        }
+        if (fieldInfo.getPointDimensionCount() == 0) {
+            throw new IllegalArgumentException("field=\"" + fieldName + "\" did not index point values");
+        }
+
+        return readers.get(fieldInfo.number);
+    }
+
+    // We only open the metadata field, and do nothing with the other two files (index/data),
+    // for which Lucene checks integrity but we don't need to.
+    @Override
+    public void checkIntegrity() {}
+
+    @Override
+    public void close() throws IOException {
+        // Free up heap:
+        readers.clear();
+    }
+}

x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene87/BWCLucene87Codec.java

@@ -0,0 +1,90 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.lucene.bwc.codecs.lucene87;
+
+import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat;
+import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat;
+import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat;
+import org.apache.lucene.backward_codecs.lucene80.Lucene80DocValuesFormat;
+import org.apache.lucene.backward_codecs.lucene86.Lucene86SegmentInfoFormat;
+import org.apache.lucene.backward_codecs.lucene87.Lucene87StoredFieldsFormat;
+import org.apache.lucene.codecs.CompoundFormat;
+import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.FieldInfosFormat;
+import org.apache.lucene.codecs.LiveDocsFormat;
+import org.apache.lucene.codecs.PointsFormat;
+import org.apache.lucene.codecs.SegmentInfoFormat;
+import org.apache.lucene.codecs.StoredFieldsFormat;
+import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
+import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec;
+import org.elasticsearch.xpack.lucene.bwc.codecs.lucene86.Lucene86MetadataOnlyPointsFormat;
+
+/**
+ * This is a fork of {@link org.apache.lucene.backward_codecs.lucene87.Lucene87Codec}
+ * Implements the Lucene 8.7 index format. Loaded via SPI for indices created/written with Lucene 8.7.0-8.11.3
+ * (Elasticsearch [7.10.0-7-17.26]), mounted as archive indices in Elasticsearch 8.x / 9.x.
+ */
+public class BWCLucene87Codec extends BWCCodec {
+
+    private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat();
+    private final CompoundFormat compoundFormat = new Lucene50CompoundFormat();
+    private final PointsFormat pointsFormat = new Lucene86MetadataOnlyPointsFormat();
+    private final DocValuesFormat defaultDVFormat;
+
+    private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() {
+        @Override
+        public DocValuesFormat getDocValuesFormatForField(String field) {
+            return defaultDVFormat;
+        }
+    };
+
+    private final StoredFieldsFormat storedFieldsFormat;
+
+    // Needed for SPI loading
+    @SuppressWarnings("unused")
+    public BWCLucene87Codec() {
+        super("BWCLucene87Codec");
+        this.storedFieldsFormat = new Lucene87StoredFieldsFormat(Lucene87StoredFieldsFormat.Mode.BEST_COMPRESSION);
+        this.defaultDVFormat = new Lucene80DocValuesFormat(Lucene80DocValuesFormat.Mode.BEST_COMPRESSION);
+    }
+
+    @Override
+    protected FieldInfosFormat originalFieldInfosFormat() {
+        return new Lucene60FieldInfosFormat();
+    }
+
+    @Override
+    protected SegmentInfoFormat originalSegmentInfoFormat() {
+        return new Lucene86SegmentInfoFormat();
+    }
+
+    @Override
+    public StoredFieldsFormat storedFieldsFormat() {
+        return storedFieldsFormat;
+    }
+
+    @Override
+    public final LiveDocsFormat liveDocsFormat() {
+        return liveDocsFormat;
+    }
+
+    @Override
+    public CompoundFormat compoundFormat() {
+        return compoundFormat;
+    }
+
+    @Override
+    public PointsFormat pointsFormat() {
+        return pointsFormat;
+    }
+
+    @Override
+    public final DocValuesFormat docValuesFormat() {
+        return docValuesFormat;
+    }
+}

x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec

@@ -5,6 +5,10 @@
 # 2.0.
 #
 
+org.elasticsearch.xpack.lucene.bwc.codecs.lucene87.BWCLucene87Codec
+org.elasticsearch.xpack.lucene.bwc.codecs.lucene86.BWCLucene86Codec
+org.elasticsearch.xpack.lucene.bwc.codecs.lucene84.BWCLucene84Codec
+org.elasticsearch.xpack.lucene.bwc.codecs.lucene80.BWCLucene80Codec
 org.elasticsearch.xpack.lucene.bwc.codecs.lucene70.BWCLucene70Codec
 org.elasticsearch.xpack.lucene.bwc.codecs.lucene70.Lucene70Codec
 org.elasticsearch.xpack.lucene.bwc.codecs.lucene62.Lucene62Codec

x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodecTests.java

@@ -0,0 +1,92 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.lucene.bwc.codecs;
+
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.CompoundFormat;
+import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.FieldInfosFormat;
+import org.apache.lucene.codecs.LiveDocsFormat;
+import org.apache.lucene.codecs.PointsFormat;
+import org.apache.lucene.codecs.PostingsFormat;
+import org.apache.lucene.codecs.SegmentInfoFormat;
+import org.apache.lucene.codecs.StoredFieldsFormat;
+import org.elasticsearch.test.ESTestCase;
+
+/**
+ * Unit tests for the {@link BWCCodec} class.
+ */
+public class BWCCodecTests extends ESTestCase {
+
+    private final Codec codec;
+
+    public BWCCodecTests() {
+        this.codec = new BWCCodec("WrapperCodec") {
+            @Override
+            protected SegmentInfoFormat originalSegmentInfoFormat() {
+                return null;
+            }
+
+            @Override
+            protected FieldInfosFormat originalFieldInfosFormat() {
+                return null;
+            }
+
+            @Override
+            public PostingsFormat postingsFormat() {
+                return null;
+            }
+
+            @Override
+            public DocValuesFormat docValuesFormat() {
+                return null;
+            }
+
+            @Override
+            public StoredFieldsFormat storedFieldsFormat() {
+                return null;
+            }
+
+            @Override
+            public LiveDocsFormat liveDocsFormat() {
+                return null;
+            }
+
+            @Override
+            public CompoundFormat compoundFormat() {
+                return null;
+            }
+
+            @Override
+            public PointsFormat pointsFormat() {
+                return null;
+            }
+        };
+    }
+
+    /**
+     * Tests that the {@link Codec#normsFormat()} method throws an {@link UnsupportedOperationException}.
+     */
+    public void testNormsFormatUnsupportedOperation() {
+        assertThrows(UnsupportedOperationException.class, codec::normsFormat);
+    }
+
+    /**
+     * Tests that the {@link Codec#termVectorsFormat()} method throws an {@link UnsupportedOperationException}.
+     */
+    public void testTermVectorsFormatUnsupportedOperation() {
+        assertThrows(UnsupportedOperationException.class, codec::termVectorsFormat);
+    }
+
+    /**
+     * Tests that the {@link Codec#knnVectorsFormat()} method throws an {@link UnsupportedOperationException}.
+     */
+    public void testKnnVectorsFormatUnsupportedOperation() {
+        assertThrows(UnsupportedOperationException.class, codec::knnVectorsFormat);
+    }
+}

x-pack/plugin/old-lucene-versions/src/test/java/org/elasticsearch/xpack/lucene/bwc/codecs/OldCodecsAvailableTests.java

@@ -7,18 +7,62 @@
 
 package org.elasticsearch.xpack.lucene.bwc.codecs;
 
+import org.apache.lucene.codecs.Codec;
 import org.elasticsearch.Version;
 import org.elasticsearch.core.UpdateForV10;
 import org.elasticsearch.test.ESTestCase;
 
+import java.util.ServiceLoader;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
 public class OldCodecsAvailableTests extends ESTestCase {
 
     /**
+     * This test verifies for each Lucene codec available via SPI; we also provide a corresponding BWC codec counterpart.
+     * Using a ServiceLoader, we fetch all classes matching the codecPathRegex (this is applied for Lucne8xCodec at the moment).
+     * For each entry of the returned list, we intend to load the BWC counterpart reflectively.
+     *
      * Reminder to add Lucene BWC codecs under {@link org.elasticsearch.xpack.lucene.bwc.codecs} whenever Elasticsearch is upgraded
      * to the next major Lucene version.
      */
     @UpdateForV10(owner = UpdateForV10.Owner.SEARCH_FOUNDATIONS)
     public void testLuceneBWCCodecsAvailable() {
-        assertEquals("Add Lucene BWC codecs for Elasticsearch version 7", 9, Version.CURRENT.major);
+        assertEquals("Add Lucene BWC codecs for Elasticsearch version 8", 9, Version.CURRENT.major);
+
+        String codecPathRegex = ".*[\\\\.](Lucene(8[0-9])Codec)";
+        Pattern codecPathPattern = Pattern.compile(codecPathRegex);
+
+        String codecClassNameRegex = "Lucene(\\d+)Codec";
+        Pattern classNamePattern = Pattern.compile(codecClassNameRegex);
+
+        for (Codec codec : ServiceLoader.load(Codec.class)) {
+            Matcher codecPathMatcher = codecPathPattern.matcher(codec.getClass().getName());
+            if (codecPathMatcher.matches()) {
+                String pathName = codec.getClass().getName();
+                int lastDotIndex = pathName.lastIndexOf('.');
+                String className = pathName.substring(lastDotIndex + 1);
+
+                Matcher classNameMatcher = classNamePattern.matcher(className);
+                if (classNameMatcher.matches()) {
+                    String codecVersion = classNameMatcher.group(1);
+                    String wrappedCodecClassPath = "org.elasticsearch.xpack.lucene.bwc.codecs.lucene"
+                        + codecVersion
+                        + ".BWCLucene"
+                        + codecVersion
+                        + "Codec";
+                    assertTrue(isClassPresent(wrappedCodecClassPath));
+                }
+            }
+        }
+    }
+
+    private static boolean isClassPresent(String className) {
+        try {
+            Class.forName(className);
+            return true;
+        } catch (ClassNotFoundException e) {
+            return false;
+        }
     }
 }

x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/AbstractUpgradeCompatibilityTestCase.java

@@ -14,8 +14,11 @@ import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering;
 
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.WarningsHandler;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.repositories.fs.FsRepository;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
@@ -37,6 +40,7 @@ import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Comparator;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.function.Consumer;
 import java.util.stream.Stream;
@@ -50,7 +54,7 @@ import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
 
 @TestCaseOrdering(AbstractUpgradeCompatibilityTestCase.TestCaseOrdering.class)
-public abstract class AbstractUpgradeCompatibilityTestCase extends ESRestTestCase {
+public class AbstractUpgradeCompatibilityTestCase extends ESRestTestCase {
 
     protected static final Version VERSION_MINUS_2 = fromString(System.getProperty("tests.minimum.index.compatible"));
     protected static final Version VERSION_MINUS_1 = fromString(System.getProperty("tests.minimum.wire.compatible"));
@@ -75,17 +79,9 @@ public abstract class AbstractUpgradeCompatibilityTestCase extends ESRestTestCas
     private static boolean upgradeFailed = false;
 
     private final Version clusterVersion;
-    private final String indexCreatedVersion;
-    private final Consumer<List<String>> warningsConsumer;
 
-    public AbstractUpgradeCompatibilityTestCase(
-        @Name("cluster") Version clusterVersion,
-        String indexCreatedVersion,
-        Consumer<List<String>> warningsConsumer
-    ) {
+    public AbstractUpgradeCompatibilityTestCase(@Name("cluster") Version clusterVersion) {
         this.clusterVersion = clusterVersion;
-        this.indexCreatedVersion = indexCreatedVersion;
-        this.warningsConsumer = warningsConsumer;
     }
 
     @ParametersFactory
@@ -131,7 +127,7 @@ public abstract class AbstractUpgradeCompatibilityTestCase extends ESRestTestCas
         assumeFalse("Cluster upgrade failed", upgradeFailed);
     }
 
-    protected static Version clusterVersion() throws Exception {
+    private static Version clusterVersion() throws Exception {
         var response = assertOK(client().performRequest(new Request("GET", "/")));
         var responseBody = createFromResponse(response);
         var version = Version.fromString(responseBody.evaluate("version.number").toString());
@@ -151,43 +147,99 @@ public abstract class AbstractUpgradeCompatibilityTestCase extends ESRestTestCas
         }
     }
 
-    protected final void verifyCompatibility(String version, Consumer<List<String>> warningsConsumer) throws Exception {
-        final String repository = "repository";
-        final String snapshot = "snapshot";
-        final String index = "index";
-        final int numDocs = 5;
+    /**
+     * This method executes in two phases. For cluster in version Current-1, restores and mounts an index snapshot and performs assertions.
+     * For cluster in version Current, asserts the previously restored/mounted index exists in the upgraded setup.
+     *
+     * @param extension  The snapshot suffix in resources covering different scenarios.
+     * @throws Exception
+     */
+    protected final void verifyCompatibility(String extension) throws Exception {
+        final String repository = "repository_" + extension;
+        final String index = "index_" + extension;
+        final String indexMount = index + "_" + repository;
+        final int numDocs = 1;
 
         String repositoryPath = REPOSITORY_PATH.getRoot().getPath();
 
         if (VERSION_MINUS_1.equals(clusterVersion())) {
             assertEquals(VERSION_MINUS_1, clusterVersion());
-            assertTrue(getIndices(client()).isEmpty());
-
-            // Copy a snapshot of an index with 5 documents
-            copySnapshotFromResources(repositoryPath, version);
-            registerRepository(client(), repository, FsRepository.TYPE, true, Settings.builder().put("location", repositoryPath).build());
-            recover(client(), repository, snapshot, index, warningsConsumer);
+            restoreMountAndAssertSnapshot(extension, repository, repositoryPath, index, indexMount, numDocs, o -> {});
+        }
 
+        if (VERSION_CURRENT.equals(clusterVersion())) {
+            assertEquals(VERSION_CURRENT, clusterVersion());
             assertTrue(getIndices(client()).contains(index));
             assertDocCount(client(), index, numDocs);
 
-            return;
+            assertTrue(getIndices(client()).contains(indexMount));
+            assertDocCount(client(), indexMount, numDocs);
         }
+    }
+
+    /**
+     * This method executes in two phases. For cluster in version Current-1 it does not do anything. For cluster in version Current,
+     * restores and mounts an index snapshot and performs assertions. The test is performed only for the Current cluster.
+     *
+     * @param extension  The snapshot suffix in resources covering different scenarios.
+     * @throws Exception
+     */
+    protected final void verifyCompatibilityNoUpgrade(String extension) throws Exception {
+        verifyCompatibilityNoUpgrade(extension, o -> {});
+    }
+
+    protected final void verifyCompatibilityNoUpgrade(String extension, Consumer<List<String>> warningsConsumer) throws Exception {
 
         if (VERSION_CURRENT.equals(clusterVersion())) {
+            String repository = "repository_" + extension;
+            String index = "index_" + extension;
+            String indexMount = index + "_" + repository;
+            int numDocs = 1;
+
             assertEquals(VERSION_CURRENT, clusterVersion());
-            assertTrue(getIndices(client()).contains(index));
-            assertDocCount(client(), index, numDocs);
+            String repositoryPath = REPOSITORY_PATH.getRoot().getPath();
+            restoreMountAndAssertSnapshot(extension, repository, repositoryPath, index, indexMount, numDocs, warningsConsumer);
         }
     }
 
-    protected abstract void recover(
-        RestClient restClient,
+    private void restoreMountAndAssertSnapshot(
+        String extension,
         String repository,
-        String snapshot,
+        String repositoryPath,
         String index,
+        String indexMount,
+        int numDocs,
         Consumer<List<String>> warningsConsumer
-    ) throws Exception;
+    ) throws Exception {
+        copySnapshotFromResources(repositoryPath, extension);
+        registerRepository(client(), repository, FsRepository.TYPE, true, Settings.builder().put("location", repositoryPath).build());
+
+        restore(client(), repository, index, warningsConsumer);
+        mount(client(), repository, index, indexMount, o -> {});
+
+        assertTrue(getIndices(client()).contains(index));
+        assertDocCount(client(), index, numDocs);
+        assertPhraseQuery(client(), index, "Elasticsearch Doc");
+
+        assertTrue(getIndices(client()).contains(indexMount));
+    }
+
+    private static void assertPhraseQuery(RestClient client, String indexName, String phrase) throws IOException {
+        var request = new Request("GET", "/" + indexName + "/_search");
+        request.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE));
+        request.setJsonEntity(Strings.format("""
+            {
+              "query": {
+                "match": {
+                  "content": "%s"
+                }
+              }
+            }""", phrase));
+        Response response = client.performRequest(request);
+        Map<String, Object> map = responseAsMap(response);
+        int hits = ((List<?>) ((Map<?, ?>) map.get("hits")).get("hits")).size();
+        assertEquals("expected 1  documents but it was a different number", 1, hits);
+    }
 
     private static String getIndices(RestClient client) throws IOException {
         final Request request = new Request("GET", "_cat/indices");
@@ -195,6 +247,38 @@ public abstract class AbstractUpgradeCompatibilityTestCase extends ESRestTestCas
         return EntityUtils.toString(response.getEntity());
     }
 
+    private void restore(RestClient client, String repository, String index, Consumer<List<String>> warningsConsumer) throws Exception {
+        var request = new Request("POST", "/_snapshot/" + repository + "/snapshot/_restore");
+        request.addParameter("wait_for_completion", "true");
+        request.setJsonEntity(Strings.format("""
+            {
+              "indices": "%s",
+              "include_global_state": false,
+              "rename_pattern": "(.+)",
+              "include_aliases": false
+            }""", index));
+        request.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE));
+        Response response = client.performRequest(request);
+        assertOK(response);
+        warningsConsumer.accept(response.getWarnings());
+    }
+
+    private void mount(RestClient client, String repository, String index, String indexMount, Consumer<List<String>> warningsConsumer)
+        throws Exception {
+        var request = new Request("POST", "/_snapshot/" + repository + "/snapshot/_mount");
+        request.addParameter("wait_for_completion", "true");
+        request.addParameter("storage", "full_copy");
+        request.setJsonEntity(Strings.format("""
+             {
+              "index": "%s",
+              "renamed_index": "%s"
+            }""", index, indexMount));
+        request.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE));
+        Response response = client.performRequest(request);
+        assertOK(response);
+        warningsConsumer.accept(response.getWarnings());
+    }
+
     private static void copySnapshotFromResources(String repositoryPath, String version) throws IOException, URISyntaxException {
         Path zipFilePath = Paths.get(
             Objects.requireNonNull(AbstractUpgradeCompatibilityTestCase.class.getClassLoader().getResource("snapshot_v" + version + ".zip"))
@@ -225,7 +309,29 @@ public abstract class AbstractUpgradeCompatibilityTestCase extends ESRestTestCas
         }
     }
 
-    public final void testArchiveIndex() throws Exception {
-        verifyCompatibility(indexCreatedVersion, warningsConsumer);
+    protected static final class TestSnapshotCases {
+        // Index created in vES_5 - Basic mapping
+        public static final String ES_VERSION_5 = "5";
+
+        // Index created in vES_5 - Custom-Analyzer - standard token filter
+        public static final String ES_VERSION_5_STANDARD_TOKEN_FILTER = "5_standard_token_filter";
+
+        // Index created in vES_6 - Basic mapping
+        public static final String ES_VERSION_6 = "6";
+
+        // Index created in vES_6 - Custom-Analyzer - standard token filter
+        public static final String ES_VERSION_6_STANDARD_TOKEN_FILTER = "6_standard_token_filter";
+
+        // Index created in vES_6 - upgraded to vES_7 LuceneCodec80
+        public static final String ES_VERSION_6_LUCENE_CODEC_80 = "lucene_80";
+
+        // Index created in vES_6 - upgraded to vES_7 LuceneCodec84
+        public static final String ES_VERSION_6_LUCENE_CODEC_84 = "lucene_84";
+
+        // Index created in vES_6 - upgraded to vES_7 LuceneCodec86
+        public static final String ES_VERSION_6_LUCENE_CODEC_86 = "lucene_86";
+
+        // Index created in vES_6 - upgraded to vES_7 LuceneCodec87
+        public static final String ES_VERSION_6_LUCENE_CODEC_87 = "lucene_87";
     }
 }

x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/OldRepositoriesIT.java

@@ -0,0 +1,175 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.oldrepos;
+
+import org.elasticsearch.test.cluster.util.Version;
+
+/**
+ * Test suite for Archive indices backward compatibility with N-2 versions.
+ * The test suite creates a cluster in the N-1 version, where N is the current version.
+ * Restores snapshots from old-clusters (version 5/6) and upgrades it to the current version.
+ * Test methods are executed after each upgrade.
+ *
+ * For example the test suite creates a cluster of version 8, then restores a snapshot of an index created
+ * when deployed ES version 5/6. The cluster then upgrades to version 9, verifying that the archive index
+ * is successfully restored.
+ */
+public class OldRepositoriesIT extends AbstractUpgradeCompatibilityTestCase {
+
+    static {
+        clusterConfig = config -> config.setting("xpack.license.self_generated.type", "trial");
+    }
+
+    public OldRepositoriesIT(Version version) {
+        super(version);
+    }
+
+    /**
+     * Test case restoring a snapshot created in ES_v5 - Basic mapping
+     * 1. Index Created in ES_v5
+     * 2. Added 1 documents to index and created a snapshot (Steps 1-2 into resources/snapshot_v5.zip)
+     * 3. Index Restored to version: Current-1: 8.x
+     * 4. Cluster upgraded to version Current: 9.x
+     */
+    public void testRestoreMountIndexVersion5() throws Exception {
+        verifyCompatibility(TestSnapshotCases.ES_VERSION_5);
+    }
+
+    /**
+     * Test case mounting a snapshot created in ES_v6 - Basic mapping
+     * 1. Index Created in ES_v6
+     * 2. Added 1 documents to index and created a snapshot (Steps 1-2 into resources/snapshot_v6.zip)
+     * 3. Index Restored to version: Current-1: 8.x
+     * 4. Cluster upgraded to version Current: 9.x
+     */
+    public void testRestoreMountIndexVersion6() throws Exception {
+        verifyCompatibility(TestSnapshotCases.ES_VERSION_6);
+    }
+
+    /**
+     * Test case restoring snapshot created in ES_v5 - Custom-analyzer including a standard-token-filter
+     {
+     *   "settings": {
+     *     "analysis": {
+     *       "analyzer": {
+     *         "custom_analyzer": {
+     *           "type": "custom",
+     *           "tokenizer": "standard",
+     *           "filter": [
+     *             "standard",
+     *             "lowercase"
+     *           ]
+     *         }
+     *       }
+     *     }
+     *   },
+     *   "mappings": {
+     *     "my_type": {
+     *       "properties": {
+     *         "content": {
+     *           "type": "text",
+     *           "analyzer": "custom_analyzer"
+     *         }
+     *       }
+     *     }
+     *   }
+     * }
+     * 1. Index Created in ES_v5
+     * 2. Added 1 documents to index and created a snapshot (Steps 1-2 into resources/snapshot_v5_standard_token_filter.zip)
+     * 3. Index Restored to version: Current: 9.x
+     */
+    public void testRestoreMountVersion5StandardTokenFilter() throws Exception {
+        verifyCompatibilityNoUpgrade(TestSnapshotCases.ES_VERSION_5_STANDARD_TOKEN_FILTER, warnings -> {
+            assertEquals(1, warnings.size());
+            assertEquals("The [standard] token filter is " + "deprecated and will be removed in a future version.", warnings.getFirst());
+        });
+    }
+
+    /**
+     * Test case restoring snapshot created in ES_v6 - Custom-analyzer including a standard-token-filter
+     {
+     *   "settings": {
+     *     "analysis": {
+     *       "analyzer": {
+     *         "custom_analyzer": {
+     *           "type": "custom",
+     *           "tokenizer": "standard",
+     *           "filter": [
+     *             "standard",
+     *             "lowercase"
+     *           ]
+     *         }
+     *       }
+     *     }
+     *   },
+     *   "mappings": {
+     *     "my_type": {
+     *       "properties": {
+     *         "content": {
+     *           "type": "text",
+     *           "analyzer": "custom_analyzer"
+     *         }
+     *       }
+     *     }
+     *   }
+     * }
+     * 1. Index Created in ES_v6
+     * 2. Added 1 documents to index and created a snapshot (Steps 1-2 into resources/snapshot_v6_standard_token_filter.zip)
+     * 3. Index Restored to version: Current: 9.x
+     */
+    public void testRestoreMountVersion6StandardTokenFilter() throws Exception {
+        verifyCompatibilityNoUpgrade(TestSnapshotCases.ES_VERSION_6_STANDARD_TOKEN_FILTER, warnings -> {
+            assertEquals(1, warnings.size());
+            assertEquals("The [standard] token filter is " + "deprecated and will be removed in a future version.", warnings.getFirst());
+        });
+    }
+
+    /**
+     * Test case restoring snapshot created with luceneCoded8.0
+     * 1. Index Created in ES_v6
+     * 2. Cluster upgraded to ES_v7.0.0 -> LuceneVersion 8.0.0 -> LuceneCodec lucene80
+     * 3. Added 1 documents to index and created a snapshot (Steps 1-3 into resources/snapshot_vlucene_80.zip)
+     * 4. Index Restored to version: Current : 9.x
+     */
+    public void testRestoreMountVersion6LuceneCodec80() throws Exception {
+        verifyCompatibilityNoUpgrade(TestSnapshotCases.ES_VERSION_6_LUCENE_CODEC_80);
+    }
+
+    /**
+     * Test case restoring snapshot created with luceneCoded8.4
+     * 1. Index Created in ES_v6
+     * 2. Cluster upgraded to ES_v7.6.0 -> LuceneVersion 8.4.0 -> LuceneCodec lucene84
+     * 3. Added 1 documents to index and created a snapshot (Steps 1-3 into resources/snapshot_vlucene_84.zip)
+     * 4. Index Restored to version: Current: 9.x
+     */
+    public void testRestoreMountVersion6LuceneCodec84() throws Exception {
+        verifyCompatibilityNoUpgrade(TestSnapshotCases.ES_VERSION_6_LUCENE_CODEC_84);
+    }
+
+    /**
+     * Test case restoring snapshot created with luceneCoded8.4
+     * 1. Index Created in ES_v6
+     * 2. Cluster upgraded to ES_v7.9.0 -> LuceneVersion 8.6.0 -> LuceneCodec lucene86
+     * 3. Added 1 documents to index and created a snapshot (Steps 1-3 into resources/snapshot_vlucene_86.zip)
+     * 4. Index Restored to version: Current: 9.x
+     */
+    public void testRestoreMountVersion6LuceneCodec86() throws Exception {
+        verifyCompatibilityNoUpgrade(TestSnapshotCases.ES_VERSION_6_LUCENE_CODEC_86);
+    }
+
+    /**
+     * Test case restoring snapshot created with luceneCoded8.4
+     * 1. Index Created in ES_v6
+     * 2. Cluster upgraded to ES_v7.10.0 -> LuceneVersion 8.7.0 -> LuceneCodec lucene87
+     * 3. Added 1 documents to index and created a snapshot (Steps 1-3 into resources/snapshot_vlucene_87.zip)
+     * 4. Index Restored to version: Current: 9.x
+     */
+    public void testRestoreMountVersion6LuceneCodec87() throws Exception {
+        verifyCompatibilityNoUpgrade(TestSnapshotCases.ES_VERSION_6_LUCENE_CODEC_87);
+    }
+}

x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/TestSnapshotCases.java

@@ -1,26 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.oldrepos;
-
-/**
-* Snapshot zip file names for archive and searchable snapshot tests
-*/
-public final class TestSnapshotCases {
-    // Index created in vES_5 - Basic mapping
-    public static final String ES_VERSION_5 = "5";
-
-    // Index created in vES_5 - Custom-Analyzer - standard token filter
-    public static final String ES_VERSION_5_STANDARD_TOKEN_FILTER = "5_custom_analyzer";
-
-    // Index created in vES_5 - Basic mapping
-    public static final String ES_VERSION_6 = "5";
-
-    // Index created in vES_5 - Custom-Analyzer - standard token filter
-    public static final String ES_VERSION_6_STANDARD_TOKEN_FILTER = "5_custom_analyzer";
-
-}

x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/ArchiveIndexTestCase.java

@@ -1,66 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.oldrepos.archiveindex;
-
-import org.elasticsearch.client.Request;
-import org.elasticsearch.client.RequestOptions;
-import org.elasticsearch.client.Response;
-import org.elasticsearch.client.RestClient;
-import org.elasticsearch.client.WarningsHandler;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.oldrepos.AbstractUpgradeCompatibilityTestCase;
-import org.elasticsearch.test.cluster.util.Version;
-
-import java.util.List;
-import java.util.function.Consumer;
-
-/**
- * Test suite for Archive indices backward compatibility with N-2 versions.
- * The test suite creates a cluster in the N-1 version, where N is the current version.
- * Restores snapshots from old-clusters (version 5/6) and upgrades it to the current version.
- * Test methods are executed after each upgrade.
- *
- * For example the test suite creates a cluster of version 8, then restores a snapshot of an index created
- * when deployed ES version 5/6. The cluster then upgrades to version 9, verifying that the archive index
- * is successfully restored.
- */
-abstract class ArchiveIndexTestCase extends AbstractUpgradeCompatibilityTestCase {
-
-    static {
-        clusterConfig = config -> config.setting("xpack.license.self_generated.type", "trial");
-    }
-
-    protected ArchiveIndexTestCase(Version version, String indexCreatedVersion) {
-        this(version, indexCreatedVersion, o -> {});
-    }
-
-    protected ArchiveIndexTestCase(Version version, String indexCreatedVersion, Consumer<List<String>> consumer) {
-        super(version, indexCreatedVersion, consumer);
-    }
-
-    /**
-     * Overrides the snapshot-restore operation for archive-indices scenario.
-     */
-    @Override
-    public void recover(RestClient client, String repository, String snapshot, String index, Consumer<List<String>> warningsConsumer)
-        throws Exception {
-        var request = new Request("POST", "/_snapshot/" + repository + "/" + snapshot + "/_restore");
-        request.addParameter("wait_for_completion", "true");
-        request.setJsonEntity(Strings.format("""
-            {
-              "indices": "%s",
-              "include_global_state": false,
-              "rename_pattern": "(.+)",
-              "include_aliases": false
-            }""", index));
-        request.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE));
-        Response response = client.performRequest(request);
-        assertOK(response);
-        warningsConsumer.accept(response.getWarnings());
-    }
-}

x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion5CustomAnalyzerIT.java

@@ -1,52 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.oldrepos.archiveindex;
-
-import org.elasticsearch.oldrepos.TestSnapshotCases;
-import org.elasticsearch.test.cluster.util.Version;
-
-/**
- * Test case restoring index created in ES_v5 - Custom-Analyzer - standard token filter
- *
- * PUT /index
- * {
- *   "settings": {
- *     "analysis": {
- *       "analyzer": {
- *         "custom_analyzer": {
- *           "type": "custom",
- *           "tokenizer": "standard",
- *           "filter": [
- *             "standard",
- *             "lowercase"
- *           ]
- *         }
- *       }
- *     }
- *   },
- *   "mappings": {
- *     "my_type": {
- *       "properties": {
- *         "content": {
- *           "type": "text",
- *           "analyzer": "custom_analyzer"
- *         }
- *       }
- *     }
- *   }
- * }
- */
-public class RestoreFromVersion5CustomAnalyzerIT extends ArchiveIndexTestCase {
-
-    public RestoreFromVersion5CustomAnalyzerIT(Version version) {
-        super(version, TestSnapshotCases.ES_VERSION_5_STANDARD_TOKEN_FILTER, warnings -> {
-            assertEquals(1, warnings.size());
-            assertEquals("The [standard] token filter is " + "deprecated and will be removed in a future version.", warnings.getFirst());
-        });
-    }
-}

x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion5IT.java

@@ -1,44 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.oldrepos.archiveindex;
-
-import org.elasticsearch.oldrepos.TestSnapshotCases;
-import org.elasticsearch.test.cluster.util.Version;
-
-/**
- * Test case restoring snapshot created in ES_v5 - Basic mapping
- *
- * PUT /index
- * {
- *   "settings": {
- *     "number_of_shards": 1,
- *     "number_of_replicas": 1
- *   },
- *   "mappings": {
- *     "my_type": {
- *       "properties": {
- *         "title": {
- *           "type": "text"
- *         },
- *         "created_at": {
- *           "type": "date"
- *         },
- *         "views": {
- *           "type": "integer"
- *         }
- *       }
- *     }
- *   }
- * }
- */
-public class RestoreFromVersion5IT extends ArchiveIndexTestCase {
-
-    public RestoreFromVersion5IT(Version version) {
-        super(version, TestSnapshotCases.ES_VERSION_5);
-    }
-}

x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion6CustomAnalyzerIT.java

@@ -1,52 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.oldrepos.archiveindex;
-
-import org.elasticsearch.oldrepos.TestSnapshotCases;
-import org.elasticsearch.test.cluster.util.Version;
-
-/**
- * Test case restoring index created in ES_v6 - Custom-Analyzer - standard token filter
- *
- * PUT /index
- * {
- *   "settings": {
- *     "analysis": {
- *       "analyzer": {
- *         "custom_analyzer": {
- *           "type": "custom",
- *           "tokenizer": "standard",
- *           "filter": [
- *             "standard",
- *             "lowercase"
- *           ]
- *         }
- *       }
- *     }
- *   },
- *   "mappings": {
- *     "_doc": {
- *       "properties": {
- *         "content": {
- *           "type": "text",
- *           "analyzer": "custom_analyzer"
- *         }
- *       }
- *     }
- *   }
- * }
- */
-public class RestoreFromVersion6CustomAnalyzerIT extends ArchiveIndexTestCase {
-
-    public RestoreFromVersion6CustomAnalyzerIT(Version version) {
-        super(version, TestSnapshotCases.ES_VERSION_6_STANDARD_TOKEN_FILTER, warnings -> {
-            assertEquals(1, warnings.size());
-            assertEquals("The [standard] token filter is " + "deprecated and will be removed in a future version.", warnings.getFirst());
-        });
-    }
-}

x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion6IT.java

@@ -1,44 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.oldrepos.archiveindex;
-
-import org.elasticsearch.oldrepos.TestSnapshotCases;
-import org.elasticsearch.test.cluster.util.Version;
-
-/**
- * Test case restoring snapshot created in ES_v6 - Basic mapping
- *
- * PUT /index
- * {
- *   "settings": {
- *     "number_of_shards": 1,
- *     "number_of_replicas": 1
- *   },
- *   "mappings": {
- *     "_doc": {
- *       "properties": {
- *         "title": {
- *           "type": "text"
- *         },
- *         "content": {
- *           "type": "text"
- *         },
- *         "created_at": {
- *           "type": "date"
- *         }
- *       }
- *     }
- *   }
- * }
- */
-public class RestoreFromVersion6IT extends ArchiveIndexTestCase {
-
-    public RestoreFromVersion6IT(Version version) {
-        super(version, TestSnapshotCases.ES_VERSION_6);
-    }
-}

x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion5CustomAnalyzerIT.java

@@ -1,52 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.oldrepos.searchablesnapshot;
-
-import org.elasticsearch.oldrepos.TestSnapshotCases;
-import org.elasticsearch.test.cluster.util.Version;
-
-/**
- * Test case mounting index created in ES_v5 - Custom-Analyzer - standard token filter
- *
- * PUT /index
- * {
- *   "settings": {
- *     "analysis": {
- *       "analyzer": {
- *         "custom_analyzer": {
- *           "type": "custom",
- *           "tokenizer": "standard",
- *           "filter": [
- *             "standard",
- *             "lowercase"
- *           ]
- *         }
- *       }
- *     }
- *   },
- *   "mappings": {
- *     "my_type": {
- *       "properties": {
- *         "content": {
- *           "type": "text",
- *           "analyzer": "custom_analyzer"
- *         }
- *       }
- *     }
- *   }
- * }
- */
-public class MountFromVersion5CustomAnalyzerIT extends SearchableSnapshotTestCase {
-
-    public MountFromVersion5CustomAnalyzerIT(Version version) {
-        super(version, TestSnapshotCases.ES_VERSION_5_STANDARD_TOKEN_FILTER, warnings -> {
-            assertEquals(1, warnings.size());
-            assertEquals("The [standard] token filter is " + "deprecated and will be removed in a future version.", warnings.getFirst());
-        });
-    }
-}

x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion5IT.java

@@ -1,44 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.oldrepos.searchablesnapshot;
-
-import org.elasticsearch.oldrepos.TestSnapshotCases;
-import org.elasticsearch.test.cluster.util.Version;
-
-/**
- * Test case mounting snapshot created in ES_v5 - Basic mapping
- *
- * PUT /index
- * {
- *   "settings": {
- *     "number_of_shards": 1,
- *     "number_of_replicas": 1
- *   },
- *   "mappings": {
- *     "my_type": {
- *       "properties": {
- *         "title": {
- *           "type": "text"
- *         },
- *         "created_at": {
- *           "type": "date"
- *         },
- *         "views": {
- *           "type": "integer"
- *         }
- *       }
- *     }
- *   }
- * }
- */
-public class MountFromVersion5IT extends SearchableSnapshotTestCase {
-
-    public MountFromVersion5IT(Version version) {
-        super(version, TestSnapshotCases.ES_VERSION_5);
-    }
-}

x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion6CustomAnalyzerIT.java

@@ -1,52 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.oldrepos.searchablesnapshot;
-
-import org.elasticsearch.oldrepos.TestSnapshotCases;
-import org.elasticsearch.test.cluster.util.Version;
-
-/**
- * Test case mounting index created in ES_v6 - Custom-Analyzer - standard token filter
- *
- * PUT /index
- * {
- *   "settings": {
- *     "analysis": {
- *       "analyzer": {
- *         "custom_analyzer": {
- *           "type": "custom",
- *           "tokenizer": "standard",
- *           "filter": [
- *             "standard",
- *             "lowercase"
- *           ]
- *         }
- *       }
- *     }
- *   },
- *   "mappings": {
- *     "_doc": {
- *       "properties": {
- *         "content": {
- *           "type": "text",
- *           "analyzer": "custom_analyzer"
- *         }
- *       }
- *     }
- *   }
- * }
- */
-public class MountFromVersion6CustomAnalyzerIT extends SearchableSnapshotTestCase {
-
-    public MountFromVersion6CustomAnalyzerIT(Version version) {
-        super(version, TestSnapshotCases.ES_VERSION_6_STANDARD_TOKEN_FILTER, warnings -> {
-            assertEquals(1, warnings.size());
-            assertEquals("The [standard] token filter is " + "deprecated and will be removed in a future version.", warnings.getFirst());
-        });
-    }
-}

x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/MountFromVersion6IT.java

@@ -1,44 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.oldrepos.searchablesnapshot;
-
-import org.elasticsearch.oldrepos.TestSnapshotCases;
-import org.elasticsearch.test.cluster.util.Version;
-
-/**
- * Test case mounting snapshot created in ES_v6 - Basic mapping
- *
- * PUT /index
- * {
- *   "settings": {
- *     "number_of_shards": 1,
- *     "number_of_replicas": 1
- *   },
- *   "mappings": {
- *     "_doc": {
- *       "properties": {
- *         "title": {
- *           "type": "text"
- *         },
- *         "content": {
- *           "type": "text"
- *         },
- *         "created_at": {
- *           "type": "date"
- *         }
- *       }
- *     }
- *   }
- * }
- */
-public class MountFromVersion6IT extends SearchableSnapshotTestCase {
-
-    public MountFromVersion6IT(Version version) {
-        super(version, TestSnapshotCases.ES_VERSION_6);
-    }
-}

x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/searchablesnapshot/SearchableSnapshotTestCase.java

@@ -1,61 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.oldrepos.searchablesnapshot;
-
-import org.elasticsearch.client.Request;
-import org.elasticsearch.client.RequestOptions;
-import org.elasticsearch.client.Response;
-import org.elasticsearch.client.RestClient;
-import org.elasticsearch.client.WarningsHandler;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.oldrepos.AbstractUpgradeCompatibilityTestCase;
-import org.elasticsearch.test.cluster.util.Version;
-
-import java.util.List;
-import java.util.function.Consumer;
-
-/**
- * Test suite for Searchable indices backward compatibility with N-2 versions.
- * The test suite creates a cluster in the N-1 version, where N is the current version.
- * Restores snapshots from old-clusters (version 5/6) and upgrades it to the current version.
- * Test methods are executed after each upgrade.
- */
-abstract class SearchableSnapshotTestCase extends AbstractUpgradeCompatibilityTestCase {
-
-    static {
-        clusterConfig = config -> config.setting("xpack.license.self_generated.type", "trial");
-    }
-
-    protected SearchableSnapshotTestCase(Version version, String indexCreatedVersion) {
-        this(version, indexCreatedVersion, o -> {});
-    }
-
-    protected SearchableSnapshotTestCase(Version version, String indexCreatedVersion, Consumer<List<String>> warningsConsumer) {
-        super(version, indexCreatedVersion, warningsConsumer);
-    }
-
-    /**
-     * Overrides the snapshot-restore operation for archive-indices scenario.
-     */
-    @Override
-    public void recover(RestClient client, String repository, String snapshot, String index, Consumer<List<String>> warningsConsumer)
-        throws Exception {
-        var request = new Request("POST", "/_snapshot/" + repository + "/" + snapshot + "/_mount");
-        request.addParameter("wait_for_completion", "true");
-        request.addParameter("storage", "full_copy");
-        request.setJsonEntity(Strings.format("""
-             {
-              "index": "%s",
-              "renamed_index": "%s"
-            }""", index, index));
-        request.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE));
-        Response response = client.performRequest(request);
-        assertOK(response);
-        warningsConsumer.accept(response.getWarnings());
-    }
-}

x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/README.md

@@ -124,7 +124,7 @@ PUT /index
 }
 ```
 
-### Create Index Version 5 - Custom-Analyzer
+### Create Index Version 6 - Custom-Analyzer
 ```
 PUT /index
 {