Address failures in SnapshotLifecycleRestIT.testFullPolicySnap… (#51055)

This test failed a couple of different ways, related to timing, as well
as concurrent snapshots, and also naming.

This commit splits the giant `assertBusy` into separate parts so that we don't
perform ~5 different requests and tests in the same loop. It also gives each
test a unique repository so that no other test can accidentally re-use
snapshots.

Resolves #50358 (hopefully!)

x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java

@@ -82,11 +82,10 @@ public class SnapshotLifecycleRestIT extends ESRestTestCase {
     }
 
     @SuppressWarnings("unchecked")
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/50358")
     public void testFullPolicySnapshot() throws Exception {
         final String indexName = "test";
         final String policyName = "test-policy";
-        final String repoId = "my-repo";
+        final String repoId = "full-policy-repo";
         int docCount = randomIntBetween(10, 50);
         List<IndexRequestBuilder> indexReqs = new ArrayList<>();
         for (int i = 0; i < docCount; i++) {
@@ -116,9 +115,12 @@ public class SnapshotLifecycleRestIT extends ESRestTestCase {
             Map<String, Object> metadata = (Map<String, Object>) snapResponse.get(0).get("metadata");
             assertNotNull(metadata);
             assertThat(metadata.get("policy"), equalTo(policyName));
-            assertHistoryIsPresent(policyName, true, repoId, CREATE_OPERATION);
+        });
+
+        assertBusy(() -> assertHistoryIsPresent(policyName, true, repoId, CREATE_OPERATION));
 
-            // Check that the last success date was written to the cluster state
+        // Check that the last success date was written to the cluster state
+        assertBusy(() -> {
             Request getReq = new Request("GET", "/_slm/policy/" + policyName);
             Response policyMetadata = client().performRequest(getReq);
             Map<String, Object> policyResponseMap;
@@ -136,9 +138,10 @@ public class SnapshotLifecycleRestIT extends ESRestTestCase {
 
             String lastSnapshotName = (String) lastSuccessObject.get("snapshot_name");
             assertThat(lastSnapshotName, startsWith("snap-"));
+        });
 
-            assertHistoryIsPresent(policyName, true, repoId, CREATE_OPERATION);
-
+        // Check that the stats are written
+        assertBusy(() -> {
             Map<String, Object> stats = getSLMStats();
             Map<String, Object> policyStats = policyStatsAsMap(stats);
             Map<String, Object> policyIdStats = (Map<String, Object>) policyStats.get(policyName);
@@ -155,7 +158,7 @@ public class SnapshotLifecycleRestIT extends ESRestTestCase {
     @SuppressWarnings("unchecked")
     public void testPolicyFailure() throws Exception {
         final String policyName = "test-policy";
-        final String repoName = "test-repo";
+        final String repoName = "policy-failure-repo";
         final String indexPattern = "index-doesnt-exist";
         initializeRepo(repoName);
 
@@ -204,7 +207,7 @@ public class SnapshotLifecycleRestIT extends ESRestTestCase {
     public void testPolicyManualExecution() throws Exception {
         final String indexName = "test";
         final String policyName = "test-policy";
-        final String repoId = "my-repo";
+        final String repoId = "manual-execution-repo";
         int docCount = randomIntBetween(10, 50);
         for (int i = 0; i < docCount; i++) {
             index(client(), indexName, "" + i, "foo", "bar");
@@ -348,10 +351,10 @@ public class SnapshotLifecycleRestIT extends ESRestTestCase {
     @SuppressWarnings("unchecked")
     @TestIssueLogging(value = "org.elasticsearch.xpack.slm:TRACE,org.elasticsearch.xpack.core.slm:TRACE,org.elasticsearch.snapshots:TRACE",
         issueUrl = "https://github.com/elastic/elasticsearch/issues/48017")
-    public void testBasicTimeBasedRetenion() throws Exception {
+    public void testBasicTimeBasedRetention() throws Exception {
         final String indexName = "test";
         final String policyName = "test-policy";
-        final String repoId = "my-repo";
+        final String repoId = "time-based-retention-repo";
         int docCount = randomIntBetween(10, 50);
         List<IndexRequestBuilder> indexReqs = new ArrayList<>();
         for (int i = 0; i < docCount; i++) {