Added metric for cache eviction of entries with non zero frequency (#100570)

The metric is called `elasticsearch.blob_cache.count_of_evicted_used_regions` 
and is incremented in both maybeEvict and forceEvict if the entry was evicted 
and the frequency count (entry.freq) is greater than 0.

docs/changelog/100570.yaml

@@ -0,0 +1,5 @@
+pr: 100570
+summary: Added metric for cache eviction of entries with non zero frequency
+area: Search
+type: enhancement
+issues: []

x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/BlobCacheMetrics.java

@@ -14,6 +14,7 @@ import org.elasticsearch.telemetry.metric.MeterRegistry;
 
 public class BlobCacheMetrics {
     private final LongCounter cacheMissCounter;
+    private final LongCounter evictedCountNonZeroFrequency;
     private final LongHistogram cacheMissLoadTimes;
 
     public BlobCacheMetrics(MeterRegistry meterRegistry) {
@@ -23,6 +24,11 @@ public class BlobCacheMetrics {
                 "The number of times there was a cache miss that triggered a read from the blob store",
                 "count"
             ),
+            meterRegistry.registerLongCounter(
+                "elasticsearch.blob_cache.count_of_evicted_used_regions",
+                "The number of times a cache entry was evicted where the frequency was not zero",
+                "entries"
+            ),
             meterRegistry.registerLongHistogram(
                 "elasticsearch.blob_cache.cache_miss_load_times",
                 "The timing data for populating entries in the blob store resulting from a cache miss.",
@@ -31,8 +37,9 @@ public class BlobCacheMetrics {
         );
     }
 
-    BlobCacheMetrics(LongCounter cacheMissCounter, LongHistogram cacheMissLoadTimes) {
+    BlobCacheMetrics(LongCounter cacheMissCounter, LongCounter evictedCountNonZeroFrequency, LongHistogram cacheMissLoadTimes) {
         this.cacheMissCounter = cacheMissCounter;
+        this.evictedCountNonZeroFrequency = evictedCountNonZeroFrequency;
         this.cacheMissLoadTimes = cacheMissLoadTimes;
     }
 
@@ -42,6 +49,10 @@ public class BlobCacheMetrics {
         return cacheMissCounter;
     }
 
+    public LongCounter getEvictedCountNonZeroFrequency() {
+        return evictedCountNonZeroFrequency;
+    }
+
     public LongHistogram getCacheMissLoadTimes() {
         return cacheMissLoadTimes;
     }

x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java

@@ -1057,18 +1057,24 @@ public class SharedBlobCacheService<KeyType> implements Releasable {
                 }
             });
             var evictedCount = 0;
+            var nonZeroFrequencyEvictedCount = 0;
             if (matchingEntries.isEmpty() == false) {
                 synchronized (SharedBlobCacheService.this) {
                     for (LFUCacheEntry entry : matchingEntries) {
+                        int frequency = entry.freq;
                         boolean evicted = entry.chunk.forceEvict();
                         if (evicted && entry.chunk.io != null) {
                             unlink(entry);
                             keyMapping.remove(entry.chunk.regionKey, entry);
                             evictedCount++;
+                            if (frequency > 0) {
+                                nonZeroFrequencyEvictedCount++;
+                            }
                         }
                     }
                 }
             }
+            blobCacheMetrics.getEvictedCountNonZeroFrequency().incrementBy(nonZeroFrequencyEvictedCount);
             return evictedCount;
         }
 
@@ -1088,8 +1094,12 @@ public class SharedBlobCacheService<KeyType> implements Releasable {
                 assignToSlot(entry, freeSlot);
             } else {
                 // need to evict something
+                int frequency;
                 synchronized (SharedBlobCacheService.this) {
-                    maybeEvict();
+                    frequency = maybeEvict();
+                }
+                if (frequency > 0) {
+                    blobCacheMetrics.getEvictedCountNonZeroFrequency().increment();
                 }
                 final SharedBytes.IO freeSlotRetry = freeRegions.poll();
                 if (freeSlotRetry != null) {
@@ -1221,18 +1231,25 @@ public class SharedBlobCacheService<KeyType> implements Releasable {
             assert invariant(entry, false);
         }
 
-        private void maybeEvict() {
+        /**
+         * Cycles through the {@link LFUCacheEntry} from 0 to max frequency and
+         * tries to evict a chunk if no one is holding onto its resources anymore
+         *
+         * @return the frequency of the evicted entry as integer or -1 if no entry was evicted from cache
+         */
+        private int maybeEvict() {
             assert Thread.holdsLock(SharedBlobCacheService.this);
-            for (int i = 0; i < maxFreq; i++) {
-                for (LFUCacheEntry entry = freqs[i]; entry != null; entry = entry.next) {
+            for (int currentFreq = 0; currentFreq < maxFreq; currentFreq++) {
+                for (LFUCacheEntry entry = freqs[currentFreq]; entry != null; entry = entry.next) {
                     boolean evicted = entry.chunk.tryEvict();
                     if (evicted && entry.chunk.io != null) {
                         unlink(entry);
                         keyMapping.remove(entry.chunk.regionKey, entry);
-                        return;
+                        return currentFreq;
                     }
                 }
             }
+            return -1;
         }
 
         private void computeDecay() {