Ensure FrozenEngine always applies engine-level wrapper (#76100)

FrozenEngine does not always apply the engine-level DirectoryReader wrapper. Only affects combination of source-only
snapshot repositories and shared_cache searchable snapshots. While it is technically a bug, it is such an exotic
combination of things that it might not be even worth calling out. The fix is mostly here to ensure that future wrappers
won't suffer from the same fate.

server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java

@@ -76,6 +76,7 @@ public class ReadOnlyEngine extends Engine {
 
     protected volatile TranslogStats translogStats;
     private final String commitId;
+    private final Function<DirectoryReader, DirectoryReader> readerWrapperFunction;
 
     /**
      * Creates a new ReadOnlyEngine. This ctor can also be used to open a read-only engine on top of an already opened
@@ -116,7 +117,8 @@ public class ReadOnlyEngine extends Engine {
                 this.seqNoStats = seqNoStats;
                 this.indexCommit = Lucene.getIndexCommit(lastCommittedSegmentInfos, directory);
                 this.lazilyLoadSoftDeletes = lazilyLoadSoftDeletes;
-                reader = wrapReader(open(indexCommit), readerWrapperFunction);
+                this.readerWrapperFunction = readerWrapperFunction;
+                reader = wrapReader(open(indexCommit));
                 readerManager = new ElasticsearchReaderManager(reader);
                 assert translogStats != null || obtainLock : "mutiple translogs instances should not be opened at the same time";
                 this.translogStats = translogStats != null ? translogStats : translogStats(config, lastCommittedSegmentInfos);
@@ -192,8 +194,7 @@ public class ReadOnlyEngine extends Engine {
         // reopened as an internal engine, which would be the path to fix the issue.
     }
 
-    protected final ElasticsearchDirectoryReader wrapReader(DirectoryReader reader,
-                                                    Function<DirectoryReader, DirectoryReader> readerWrapperFunction) throws IOException {
+    protected final ElasticsearchDirectoryReader wrapReader(DirectoryReader reader) throws IOException {
         reader = readerWrapperFunction.apply(reader);
         return ElasticsearchDirectoryReader.wrap(reader, engineConfig.getShardId());
     }

x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/frozen/FrozenEngine.java

@@ -161,7 +161,7 @@ public final class FrozenEngine extends ReadOnlyEngine {
                     listeners.beforeRefresh();
                 }
                 final DirectoryReader dirReader = openDirectory(engineConfig.getStore().directory());
-                reader = lastOpenedReader = wrapReader(dirReader, Function.identity());
+                reader = lastOpenedReader = wrapReader(dirReader);
                 reader.getReaderCacheHelper().addClosedListener(this::onReaderClosed);
                 for (ReferenceManager.RefreshListener listeners : config().getInternalRefreshListener()) {
                     listeners.afterRefresh(true);

x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsRestTestCase.java

@@ -21,10 +21,12 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.index.mapper.SeqNoFieldMapper;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.test.rest.ESRestTestCase;
 
 import java.io.IOException;
@@ -468,7 +470,12 @@ public abstract class AbstractSearchableSnapshotsRestTestCase extends ESRestTest
 
     protected static Map<String, Object> search(String index, QueryBuilder query, Boolean ignoreThrottled) throws IOException {
         final Request request = new Request(HttpPost.METHOD_NAME, '/' + index + "/_search");
-        request.setJsonEntity(new SearchSourceBuilder().trackTotalHits(true).query(query).toString());
+        final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().trackTotalHits(true).query(query);
+        if (randomBoolean()) {
+            // ensures we can always access sequence numbers, even on source-only mounted snapshots
+            searchSourceBuilder.sort(SeqNoFieldMapper.NAME, SortOrder.ASC);
+        }
+        request.setJsonEntity(searchSourceBuilder.toString());
         if (ignoreThrottled != null) {
             request.addParameter("ignore_throttled", ignoreThrottled.toString());
         }