Add searchable snapshot stats for reads from Lucene (#70464)

Adds information about how many bytes Lucene is requesting. This helps compare it to how many bytes are requested
from the blob store to better understand how much random access reads are happening.

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/searchablesnapshots/SearchableSnapshotShardStats.java

@@ -145,6 +145,7 @@ public class SearchableSnapshotShardStats implements Writeable, ToXContentObject
         private final TimedCounter directBytesRead;
         private final TimedCounter optimizedBytesRead;
         private final Counter blobStoreBytesRequested;
+        private final Counter luceneBytesRead;
         private final long currentIndexCacheFills;
 
         public CacheIndexInputStats(String fileExt, long numFiles, ByteSizeValue totalSize, ByteSizeValue minSize, ByteSizeValue maxSize,
@@ -154,7 +155,8 @@ public class SearchableSnapshotShardStats implements Writeable, ToXContentObject
                                     Counter contiguousReads, Counter nonContiguousReads,
                                     Counter cachedBytesRead, Counter indexCacheBytesRead,
                                     TimedCounter cachedBytesWritten, TimedCounter directBytesRead, TimedCounter optimizedBytesRead,
-                                    Counter blobStoreBytesRequested, long currentIndexCacheFills) {
+                                    Counter blobStoreBytesRequested, Counter luceneBytesRead,
+                                    long currentIndexCacheFills) {
             this.fileExt = fileExt;
             this.numFiles = numFiles;
             this.totalSize = totalSize;
@@ -174,6 +176,7 @@ public class SearchableSnapshotShardStats implements Writeable, ToXContentObject
             this.directBytesRead = directBytesRead;
             this.optimizedBytesRead = optimizedBytesRead;
             this.blobStoreBytesRequested = blobStoreBytesRequested;
+            this.luceneBytesRead = luceneBytesRead;
             this.currentIndexCacheFills = currentIndexCacheFills;
         }
 
@@ -209,6 +212,11 @@ public class SearchableSnapshotShardStats implements Writeable, ToXContentObject
             this.directBytesRead = new TimedCounter(in);
             this.optimizedBytesRead = new TimedCounter(in);
             this.blobStoreBytesRequested = new Counter(in);
+            if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+                this.luceneBytesRead = new Counter(in);
+            } else {
+                this.luceneBytesRead = new Counter(0, 0, 0, 0);
+            }
             this.currentIndexCacheFills = in.readVLong();
         }
 
@@ -238,6 +246,7 @@ public class SearchableSnapshotShardStats implements Writeable, ToXContentObject
                 cis1.directBytesRead.add(cis2.directBytesRead),
                 cis1.optimizedBytesRead.add(cis2.optimizedBytesRead),
                 cis1.blobStoreBytesRequested.add(cis2.blobStoreBytesRequested),
+                cis1.luceneBytesRead.add(cis2.luceneBytesRead),
                 cis1.currentIndexCacheFills + cis2.currentIndexCacheFills
             );
         }
@@ -274,6 +283,9 @@ public class SearchableSnapshotShardStats implements Writeable, ToXContentObject
             directBytesRead.writeTo(out);
             optimizedBytesRead.writeTo(out);
             blobStoreBytesRequested.writeTo(out);
+            if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+                luceneBytesRead.writeTo(out);
+            }
             out.writeVLong(currentIndexCacheFills);
         }
 
@@ -358,6 +370,10 @@ public class SearchableSnapshotShardStats implements Writeable, ToXContentObject
             return blobStoreBytesRequested;
         }
 
+        public Counter getLuceneBytesRead() {
+            return luceneBytesRead;
+        }
+
         public long getCurrentIndexCacheFills() {
             return currentIndexCacheFills;
         }
@@ -398,6 +414,7 @@ public class SearchableSnapshotShardStats implements Writeable, ToXContentObject
                     builder.endObject();
                 }
                 builder.field("blob_store_bytes_requested", getBlobStoreBytesRequested(), params);
+                builder.field("lucene_bytes_read", getLuceneBytesRead(), params);
                 builder.field("current_index_cache_fills", getCurrentIndexCacheFills());
             }
             return builder.endObject();
@@ -431,6 +448,7 @@ public class SearchableSnapshotShardStats implements Writeable, ToXContentObject
                 && Objects.equals(directBytesRead, stats.directBytesRead)
                 && Objects.equals(optimizedBytesRead, stats.optimizedBytesRead)
                 && Objects.equals(blobStoreBytesRequested, stats.blobStoreBytesRequested)
+                && Objects.equals(luceneBytesRead, stats.luceneBytesRead)
                 && currentIndexCacheFills == stats.currentIndexCacheFills;
         }
 
@@ -444,7 +462,7 @@ public class SearchableSnapshotShardStats implements Writeable, ToXContentObject
                 contiguousReads, nonContiguousReads,
                 cachedBytesRead, indexCacheBytesRead,
                 cachedBytesWritten, directBytesRead, optimizedBytesRead,
-                blobStoreBytesRequested, currentIndexCacheFills);
+                blobStoreBytesRequested, luceneBytesRead, currentIndexCacheFills);
         }
     }
 

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/searchablesnapshots/SearchableSnapshotShardStatsTests.java

@@ -51,7 +51,7 @@ public class SearchableSnapshotShardStatsTests extends AbstractWireSerializingTe
             randomCounter(), randomCounter(),
             randomCounter(), randomCounter(), randomTimedCounter(),
             randomTimedCounter(), randomTimedCounter(),
-            randomCounter(), randomNonNegativeLong());
+            randomCounter(), randomCounter(), randomNonNegativeLong());
     }
 
     private Counter randomCounter() {

x-pack/plugin/searchable-snapshots/qa/rest/src/test/resources/rest-api-spec/test/stats.yml

@@ -66,8 +66,8 @@ teardown:
 ---
 "Tests searchable snapshots stats":
   - skip:
-      version: " - 7.12.99"
-      reason:  searchable snapshots response format changed in 7.13.0
+      version: " - 7.99.99"
+      reason: adapt after backport
 
   - do:
       catch: missing
@@ -211,6 +211,11 @@ teardown:
   - gte:     { total.0.blob_store_bytes_requested.min: 0 }
   - gte:     { total.0.blob_store_bytes_requested.max: 0 }
 
+  - gte:     { total.0.lucene_bytes_read.count: 0 }
+  - gte:     { total.0.lucene_bytes_read.sum: 0 }
+  - gte:     { total.0.lucene_bytes_read.min: 0 }
+  - gte:     { total.0.lucene_bytes_read.max: 0 }
+
   - gte:     { total.0.current_index_cache_fills: 0 }
 
   - length:  { indices: 1 }
@@ -291,6 +296,11 @@ teardown:
   - gte:     { indices.docs.total.0.blob_store_bytes_requested.min: 0 }
   - gte:     { indices.docs.total.0.blob_store_bytes_requested.max: 0 }
 
+  - gte:     { indices.docs.total.0.lucene_bytes_read.count: 0 }
+  - gte:     { indices.docs.total.0.lucene_bytes_read.sum: 0 }
+  - gte:     { indices.docs.total.0.lucene_bytes_read.min: 0 }
+  - gte:     { indices.docs.total.0.lucene_bytes_read.max: 0 }
+
   - gte:     { indices.docs.total.0.current_index_cache_fills: 0 }
 
   - length:  { indices.docs.shards: 1 }
@@ -375,6 +385,11 @@ teardown:
   - gte:     { indices.docs.shards.0.0.files.0.blob_store_bytes_requested.min: 0 }
   - gte:     { indices.docs.shards.0.0.files.0.blob_store_bytes_requested.max: 0 }
 
+  - gte:     { indices.docs.shards.0.0.files.0.lucene_bytes_read.count: 0 }
+  - gte:     { indices.docs.shards.0.0.files.0.lucene_bytes_read.sum: 0 }
+  - gte:     { indices.docs.shards.0.0.files.0.lucene_bytes_read.min: 0 }
+  - gte:     { indices.docs.shards.0.0.files.0.lucene_bytes_read.max: 0 }
+
   - gte:     { indices.docs.shards.0.0.files.0.current_index_cache_fills: 0 }
 
   - do:

x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java

@@ -1496,6 +1496,11 @@ public class SearchableSnapshotsIntegTests extends BaseSearchableSnapshotsIntegT
                         indexInputStats.getAverageSize().getBytes(),
                         greaterThan(0L)
                     );
+                    assertThat(
+                        "Expected at least one Lucene read for " + fileExt + " of shard " + shardRouting,
+                        indexInputStats.getLuceneBytesRead().getCount(),
+                        greaterThan(0L)
+                    );
 
                     if (cacheEnabled == false || nonCachedExtensions.contains(fileExt)) {
                         assertThat(

x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/BaseSearchableSnapshotIndexInput.java

@@ -101,15 +101,16 @@ public abstract class BaseSearchableSnapshotIndexInput extends BufferedIndexInpu
     protected final void readInternal(ByteBuffer b) throws IOException {
         assert assertCurrentThreadIsNotCacheFetchAsync();
 
+        final int bytesToRead = b.remaining();
         // We can detect that we're going to read the last 16 bytes (that contains the footer checksum) of the file. Such reads are often
         // executed when opening a Directory and since we have the checksum in the snapshot metadata we can use it to fill the ByteBuffer.
         if (maybeReadChecksumFromFileInfo(b)) {
             logger.trace("read footer of file [{}], bypassing all caches", fileInfo.physicalName());
-            assert b.remaining() == 0L : b.remaining();
-            return;
+        } else {
+            doReadInternal(b);
         }
-
-        doReadInternal(b);
+        assert b.remaining() == 0L : b.remaining();
+        stats.addLuceneBytesRead(bytesToRead);
     }
 
     protected abstract void doReadInternal(ByteBuffer b) throws IOException;

x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/IndexInputStats.java

@@ -55,6 +55,8 @@ public class IndexInputStats {
     private final Counter blobStoreBytesRequested = new Counter();
     private final AtomicLong currentIndexCacheFills = new AtomicLong();
 
+    private final Counter luceneBytesRead = new Counter();
+
     public IndexInputStats(long numFiles, long totalSize, long minSize, long maxSize, LongSupplier currentTimeNanos) {
         this(numFiles, totalSize, minSize, maxSize, SEEKING_THRESHOLD.getBytes(), currentTimeNanos);
     }
@@ -149,6 +151,10 @@ public class IndexInputStats {
         };
     }
 
+    public void addLuceneBytesRead(int bytesRead) {
+        luceneBytesRead.add(bytesRead);
+    }
+
     public long getNumFiles() {
         return numFiles;
     }
@@ -221,6 +227,10 @@ public class IndexInputStats {
         return blobStoreBytesRequested;
     }
 
+    public Counter getLuceneBytesRead() {
+        return luceneBytesRead;
+    }
+
     @SuppressForbidden(reason = "Handles Long.MIN_VALUE before using Math.abs()")
     public boolean isLargeSeek(long delta) {
         return delta != Long.MIN_VALUE && Math.abs(delta) > seekingThreshold;

x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/cache/FrozenIndexInput.java

@@ -111,6 +111,7 @@ public class FrozenIndexInput extends MetadataCachingIndexInput {
     protected void readWithoutBlobCache(ByteBuffer b) throws Exception {
         final long position = getAbsolutePosition();
         final int length = b.remaining();
+        final int originalByteBufPosition = b.position();
 
         final ReentrantReadWriteLock luceneByteBufLock = new ReentrantReadWriteLock();
         final AtomicBoolean stopAsyncReads = new AtomicBoolean();
@@ -172,7 +173,7 @@ public class FrozenIndexInput extends MetadataCachingIndexInput {
             assert luceneByteBufLock.getReadHoldCount() == 0;
 
             preventAsyncBufferChanges.run();
-            b.position(bytesRead); // mark all bytes as accounted for
+            b.position(originalByteBufPosition + bytesRead); // mark all bytes as accounted for
         } finally {
             preventAsyncBufferChanges.run();
         }

x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportSearchableSnapshotsStatsAction.java

@@ -118,6 +118,7 @@ public class TransportSearchableSnapshotsStatsAction extends AbstractTransportSe
             toTimedCounter(inputStats.getDirectBytesRead()),
             toTimedCounter(inputStats.getOptimizedBytesRead()),
             toCounter(inputStats.getBlobStoreBytesRequested()),
+            toCounter(inputStats.getLuceneBytesRead()),
             inputStats.getCurrentIndexCacheFills()
         );
     }

x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/action/SearchableSnapshotsStatsResponseTests.java

@@ -117,6 +117,7 @@ public class SearchableSnapshotsStatsResponseTests extends ESTestCase {
             randomTimedCounter(),
             randomTimedCounter(),
             randomCounter(),
+            randomCounter(),
             randomNonNegativeLong()
         );
     }