[CORE] Read segment info from latest commit whenever possible

Instead of listing the directory to file the latest segments_N file, we
should re-use the generation/filename from the last commit. This allows
us to avoid potential race conditions on the filesystem as well as
reduce the number of directory listings performed.

src/main/java/org/elasticsearch/common/lucene/Lucene.java

@@ -159,8 +159,11 @@ public class Lucene {
     /**
      * Reads the segments infos from the given commit, failing if it fails to load
      */
-    public static SegmentInfos readSegmentInfos(IndexCommit commit, Directory directory) throws IOException {
-        return SegmentInfos.readCommit(directory, commit.getSegmentsFileName());
+    public static SegmentInfos readSegmentInfos(IndexCommit commit) throws IOException {
+        // Using commit.getSegmentsFileName() does NOT work here, have to
+        // manually create the segment filename
+        String filename = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", commit.getGeneration());
+        return SegmentInfos.readCommit(commit.getDirectory(), filename);
     }
 
     /**

src/main/java/org/elasticsearch/index/engine/Engine.java

@@ -21,15 +21,7 @@ package org.elasticsearch.index.engine;
 
 import com.google.common.base.Preconditions;
 
-import org.apache.lucene.index.FilterLeafReader;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.SegmentCommitInfo;
-import org.apache.lucene.index.SegmentInfos;
-import org.apache.lucene.index.SegmentReader;
-import org.apache.lucene.index.Term;
+import org.apache.lucene.index.*;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.SearcherManager;
@@ -324,7 +316,18 @@ public abstract class Engine implements Closeable {
         return new CommitStats(getLastCommittedSegmentInfos());
     }
 
-
+    /**
+     * Read the last segments info from the commit pointed to by the searcher manager
+     */
+    protected static SegmentInfos readLastCommittedSegmentInfos(SearcherManager sm) throws IOException {
+        IndexSearcher searcher = sm.acquire();
+        try {
+            IndexCommit latestCommit = ((DirectoryReader) searcher.getIndexReader()).getIndexCommit();
+            return Lucene.readSegmentInfos(latestCommit);
+        } finally {
+            sm.release(searcher);
+        }
+    }
 
     /**
      * Global stats on segments.

src/main/java/org/elasticsearch/index/engine/InternalEngine.java

@@ -275,7 +275,7 @@ public class InternalEngine extends Engine {
             try {
                 final DirectoryReader directoryReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(indexWriter, true), shardId);
                 searcherManager = new SearcherManager(directoryReader, searcherFactory);
-                lastCommittedSegmentInfos = store.readLastCommittedSegmentsInfo();
+                lastCommittedSegmentInfos = readLastCommittedSegmentInfos(searcherManager);
                 success = true;
                 return searcherManager;
             } catch (IOException e) {

src/main/java/org/elasticsearch/index/engine/ShadowEngine.java

@@ -79,7 +79,7 @@ public class ShadowEngine extends Engine {
                 if (Lucene.waitForIndex(store.directory(), nonexistentRetryTime)) {
                     reader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(store.directory()), shardId);
                     this.searcherManager = new SearcherManager(reader, searcherFactory);
-                    this.lastCommittedSegmentInfos = store.readLastCommittedSegmentsInfo();
+                    this.lastCommittedSegmentInfos = readLastCommittedSegmentInfos(searcherManager);
                     success = true;
                 } else {
                     throw new IndexShardException(shardId, "failed to open a shadow engine after" +
@@ -148,7 +148,7 @@ public class ShadowEngine extends Engine {
         store.incRef();
         try (ReleasableLock lock = readLock.acquire()) {
             // reread the last committed segment infos
-            lastCommittedSegmentInfos = store.readLastCommittedSegmentsInfo();
+            lastCommittedSegmentInfos = readLastCommittedSegmentInfos(searcherManager);
         } catch (Throwable e) {
             if (isClosed.get() == false) {
                 logger.warn("failed to read latest segment infos on flush", e);

src/main/java/org/elasticsearch/index/store/Store.java

@@ -151,8 +151,9 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
      * @throws IOException if the index is corrupted or the segments file is not present
      */
     private static SegmentInfos readSegmentsInfo(IndexCommit commit, Directory directory) throws IOException {
+        assert commit == null || commit.getDirectory() == directory;
         try {
-            return commit == null ? Lucene.readSegmentInfos(directory) : Lucene.readSegmentInfos(commit, directory);
+            return commit == null ? Lucene.readSegmentInfos(directory) : Lucene.readSegmentInfos(commit);
         } catch (EOFException eof) {
             // TODO this should be caught by lucene - EOF is almost certainly an index corruption
             throw new CorruptIndexException("Read past EOF while reading segment infos", "commit(" + commit + ")", eof);