Enhance Shard Level Metdata check in BlobStoreTestUtil (#75737)

Adding check that shard level index metadata actually contains the snapshots
it's supposed to contain. This would have caught a number of recent bugs.

server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java

@@ -453,6 +453,8 @@ public class CorruptedBlobStoreRepositoryIT extends AbstractSnapshotIntegTestCas
      * Tests that a shard snapshot with a corrupted shard index file can still be used for restore and incremental snapshots.
      */
     public void testSnapshotWithCorruptedShardIndexFile() throws Exception {
+        disableRepoConsistencyCheck("This test intentionally corrupts the repository contents");
+
         final Client client = client();
         final Path repo = randomRepoPath();
         final String indexName = "test-idx";

server/src/main/java/org/elasticsearch/repositories/IndexSnapshotsService.java

@@ -162,7 +162,7 @@ public class IndexSnapshotsService {
         private BlobStoreIndexShardSnapshots getBlobStoreIndexShardSnapshots() throws IOException {
             BlobStoreRepository blobStoreRepository = (BlobStoreRepository) repository;
             final String shardGen = repositoryData.shardGenerations().getShardGen(indexId, shardId.getId());
-            return blobStoreRepository.getBlobStoreIndexShardSnapshots(indexId, shardId, shardGen);
+            return blobStoreRepository.getBlobStoreIndexShardSnapshots(indexId, shardId.getId(), shardGen);
         }
 
         private ShardSnapshotInfo createIndexShardSnapshotInfo(String indexMetadataId, SnapshotFiles snapshotFiles) {

server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java

@@ -3238,10 +3238,9 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
      * Loads all available snapshots in the repository using the given {@code generation} for a shard. When {@code shardGen}
      * is null it tries to load it using the BwC mode, listing the available index- blobs in the shard container.
      */
-    public BlobStoreIndexShardSnapshots getBlobStoreIndexShardSnapshots(IndexId indexId, ShardId shardId, @Nullable String shardGen)
+    public BlobStoreIndexShardSnapshots getBlobStoreIndexShardSnapshots(IndexId indexId, int shardId, @Nullable String shardGen)
         throws IOException {
-        final int shard = shardId.getId();
-        final BlobContainer shardContainer = shardContainer(indexId, shard);
+        final BlobContainer shardContainer = shardContainer(indexId, shardId);
 
         Set<String> blobs = Collections.emptySet();
         if (shardGen == null) {

test/framework/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreTestUtil.java

@@ -34,6 +34,7 @@ import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshots;
 import org.elasticsearch.repositories.GetSnapshotInfoContext;
 import org.elasticsearch.repositories.IndexId;
 import org.elasticsearch.repositories.RepositoryData;
@@ -300,6 +301,10 @@ public final class BlobStoreTestUtil {
                             hasKey(String.format(Locale.ROOT, BlobStoreRepository.SNAPSHOT_NAME_FORMAT, snapshotId.getUUID())));
                         assertThat(shardPathContents.keySet().stream()
                             .filter(name -> name.startsWith(BlobStoreRepository.INDEX_FILE_PREFIX)).count(), lessThanOrEqualTo(2L));
+                        final BlobStoreIndexShardSnapshots blobStoreIndexShardSnapshots = repository.getBlobStoreIndexShardSnapshots(
+                                indexId, shardId, repositoryData.shardGenerations().getShardGen(indexId, shardId));
+                        assertTrue(blobStoreIndexShardSnapshots.snapshots().stream()
+                                .anyMatch(snapshotFiles -> snapshotFiles.snapshot().equals(snapshotId.getName())));
                     }
                 }
             }