Add extra steps to ensure that watch indices are created before test starts (#66416)

This PR updates the initialization logic in TransportMonitoringMigrateAlertsActionTests to wait 
for the watch indices separately from the rest of the initialization logic in order to allow for the 
.watches index to be available before the test begins.

x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringMigrateAlertsActionTests.java

@@ -29,6 +29,7 @@ import org.elasticsearch.xpack.core.monitoring.action.MonitoringMigrateAlertsRes
 import org.elasticsearch.xpack.core.monitoring.exporter.MonitoringTemplateUtils;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchAction;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchRequest;
+import org.elasticsearch.xpack.core.watcher.watch.Watch;
 import org.elasticsearch.xpack.monitoring.Monitoring;
 import org.elasticsearch.xpack.monitoring.MonitoringService;
 import org.elasticsearch.xpack.monitoring.exporter.ClusterAlertsUtil;
@@ -119,12 +120,7 @@ public class TransportMonitoringMigrateAlertsActionTests extends MonitoringInteg
             assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(exporterSettings));
 
             // ensure resources exist
-            assertBusy(() -> {
-                assertThat(indexExists(".monitoring-*"), is(true));
-                ensureYellowAndNoInitializingShards(".monitoring-*");
-                checkMonitoringTemplates();
-                assertWatchesExist(true);
-            });
+            ensureInitialLocalResources();
 
             // call migration api
             MonitoringMigrateAlertsResponse response = client().execute(MonitoringMigrateAlertsAction.INSTANCE,
@@ -158,12 +154,7 @@ public class TransportMonitoringMigrateAlertsActionTests extends MonitoringInteg
             assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(exporterSettings));
 
             // ensure resources exist
-            assertBusy(() -> {
-                assertThat(indexExists(".monitoring-*"), is(true));
-                ensureYellowAndNoInitializingShards(".monitoring-*");
-                checkMonitoringTemplates();
-                assertWatchesExist(true);
-            });
+            ensureInitialLocalResources();
 
             // call migration api
             MonitoringMigrateAlertsResponse response = client().execute(MonitoringMigrateAlertsAction.INSTANCE,
@@ -463,6 +454,7 @@ public class TransportMonitoringMigrateAlertsActionTests extends MonitoringInteg
     }
 
     private void ensureInitialLocalResources() throws Exception {
+        waitForWatcherIndices();
         assertBusy(() -> {
             assertThat(indexExists(".monitoring-*"), is(true));
             ensureYellowAndNoInitializingShards(".monitoring-*");
@@ -579,4 +571,9 @@ public class TransportMonitoringMigrateAlertsActionTests extends MonitoringInteg
             }
         }
     }
+
+    protected void waitForWatcherIndices() throws Exception {
+        awaitIndexExists(Watch.INDEX);
+        assertBusy(() -> ensureYellowAndNoInitializingShards(Watch.INDEX));
+    }
 }

x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MonitoringIntegTestCase.java

@@ -176,7 +176,7 @@ public abstract class MonitoringIntegTestCase extends ESIntegTestCase {
         assertBusy(this::ensureMonitoringIndicesYellow);
     }
 
-    private void awaitIndexExists(final String index) throws Exception {
+    protected void awaitIndexExists(final String index) throws Exception {
         assertBusy(() -> assertIndicesExists(index), 30, TimeUnit.SECONDS);
     }