[ML] updating rest and HLRC ML tests to wait for no initializing shards & necessary indices (#71146)

In the high level rest client tests, we should clean up trained models, same as we do for datafeeds, jobs, etc.

But, to delete a trained model, it needs to not be used in a pipeline, so we grab the model stats. 

This commit does:
- waiting for no more initializing shards when ML integration tests are cleaning up before attempting to delete things
- The specific inference stats test now waits for the `.ml-stats-00001` index to exist before continuing 
- And we make sure that config index is green before attempting to run the `pipeline_inference.yml` tests

closes https://github.com/elastic/elasticsearch/issues/71139

client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java

@@ -15,6 +15,7 @@ import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.ingest.DeletePipelineRequest;
 import org.elasticsearch.action.ingest.PutPipelineRequest;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
@@ -225,6 +226,7 @@ public class MachineLearningIT extends ESRestHighLevelClientTestCase {
 
     @After
     public void cleanUp() throws IOException {
+        ensureNoInitializingShards();
         new MlTestStateCleaner(logger, highLevelClient()).clearMlMetadata();
     }
 
@@ -2468,7 +2470,6 @@ public class MachineLearningIT extends ESRestHighLevelClientTestCase {
         assertThat(exception.status().getStatus(), equalTo(404));
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/71139")
     public void testGetTrainedModelsStats() throws Exception {
         MachineLearningClient machineLearningClient = highLevelClient().machineLearning();
         String modelIdPrefix = "a-get-trained-model-stats-";
@@ -2493,12 +2494,17 @@ public class MachineLearningIT extends ESRestHighLevelClientTestCase {
             "          }\n" +
             "        }\n" +
             "      }]}\n";
+        String pipelineId = "regression-stats-pipeline";
 
         highLevelClient().ingest().putPipeline(
-            new PutPipelineRequest("regression-stats-pipeline",
+            new PutPipelineRequest(pipelineId,
                 new BytesArray(regressionPipeline.getBytes(StandardCharsets.UTF_8)),
                 XContentType.JSON),
             RequestOptions.DEFAULT);
+        highLevelClient().index(
+            new IndexRequest("trained-models-stats-test-index").source("{\"col1\": 1}", XContentType.JSON).setPipeline(pipelineId),
+            RequestOptions.DEFAULT
+        );
         {
             GetTrainedModelsStatsResponse getTrainedModelsStatsResponse = execute(
                 GetTrainedModelsStatsRequest.getAllTrainedModelStatsRequest(),
@@ -2528,6 +2534,11 @@ public class MachineLearningIT extends ESRestHighLevelClientTestCase {
                     .collect(Collectors.toList()),
                 containsInAnyOrder(modelIdPrefix + 1, modelIdPrefix + 2));
         }
+        highLevelClient().ingest().deletePipeline(new DeletePipelineRequest(pipelineId), RequestOptions.DEFAULT);
+        assertBusy(() -> {
+            assertTrue(indexExists(".ml-stats-000001"));
+            ensureGreen(".ml-stats-*");
+        });
     }
 
     public void testDeleteTrainedModel() throws Exception {

x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/pipeline_inference.yml

@@ -99,6 +99,12 @@ setup:
           { "product": "VCR", "cost": 180, "time": 1587501233000 }
           { "index": {} }
           { "product": "Laptop", "cost": 15000, "time": 1587501233000 }
+  - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
+      cluster.health:
+        index: [".ml-inference-*", ".ml-stats-*"]
+        wait_for_status: green
 
 ---
 "Test pipeline regression simple":