Wait for pending tasks in docs tests cleanup (#44123)

ML and Data Frame tests should wait for pending tasks

docs/reference/data-frames/apis/put-transform.asciidoc

@@ -110,7 +110,7 @@ PUT _data_frame/transforms/ecommerce_transform
 }
 --------------------------------------------------
 // CONSOLE
-// TEST[skip: https://github.com/elastic/elasticsearch/issues/43271]
+// TEST[setup:kibana_sample_data_ecommerce]
 
 When the transform is created, you receive the following results:
 [source,js]

docs/reference/ml/apis/put-job.asciidoc

@@ -120,7 +120,6 @@ PUT _ml/anomaly_detectors/total-requests
 }
 --------------------------------------------------
 // CONSOLE
-// TEST[skip: https://github.com/elastic/elasticsearch/issues/43271]
 
 When the job is created, you receive the following results:
 [source,js]

docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java

@@ -33,6 +33,7 @@ import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentLocation;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
+import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.yaml.ClientYamlDocsTestClient;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestClient;
@@ -41,6 +42,7 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestSpec;
 import org.elasticsearch.test.rest.yaml.section.ExecutableSection;
+import org.junit.After;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -97,6 +99,23 @@ public class DocsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
         return new ClientYamlDocsTestClient(restSpec, restClient, hosts, esVersion, masterVersion, this::getClientBuilderWithSniffedHosts);
     }
 
+    @After
+    public void cleanup() throws Exception {
+        if (isMachineLearningTest() || isDataFrameTest()) {
+            ESRestTestCase.waitForPendingTasks(adminClient());
+        }
+    }
+
+    protected boolean isMachineLearningTest() {
+        String testName = getTestName();
+        return testName != null && (testName.contains("/ml/") || testName.contains("\\ml\\"));
+    }
+
+    protected boolean isDataFrameTest() {
+        String testName = getTestName();
+        return testName != null && (testName.contains("/data-frames/") || testName.contains("\\data-frames\\"));
+    }
+
     /**
      * Compares the results of running two analyzers against many random
      * strings. The goal is to figure out if two anlayzers are "the same" by