Stricter checks of setup and teardown in docs tests (#51430)

Made checks stricter after backporting PR.

docs/reference/cat/templates.asciidoc

@@ -71,3 +71,15 @@ template1 [tea*]         1
 template2 [teak*]        2     7
 --------------------------------------------------
 // TESTRESPONSE[s/\*/\\*/ s/\[/\\[/ s/\]/\\]/ non_json]
+
+//////////////////////////
+
+[source,console]
+--------------------------------------------------
+DELETE /_template/template0
+DELETE /_template/template1
+DELETE /_template/template2
+--------------------------------------------------
+// TEST[continued]
+
+//////////////////////////

docs/reference/ilm/getting-started-ilm.asciidoc

@@ -90,6 +90,16 @@ PUT _template/datastream_template
 -----------------------
 // TEST[continued]
 
+//////////////////////////
+
+[source,console]
+--------------------------------------------------
+DELETE /_template/datastream_template
+--------------------------------------------------
+// TEST[continued]
+
+//////////////////////////
+
 <1> match all indices starting with "datastream-". These will include all
     newly created indices from actions like rollover
 <2> the name of the lifecycle policy managing the index

docs/reference/ilm/ilm-with-existing-indices.asciidoc

@@ -65,6 +65,16 @@ POST mylogs-pre-ilm-2019.06.25/_doc
 -----------------------
 // TEST[continued]
 
+//////////////////////////
+
+[source,console]
+--------------------------------------------------
+DELETE _template/mylogs_template
+--------------------------------------------------
+// TEST[continued]
+
+//////////////////////////
+
 Now that we have these indices, we'll look at a few different ways of migrating
 these indices to ILM.
 
@@ -316,6 +326,16 @@ PUT ilm-mylogs-000001
 -----------------------
 // TEST[continued]
 
+//////////////////////////
+
+[source,console]
+--------------------------------------------------
+DELETE /_template/mylogs_template
+--------------------------------------------------
+// TEST[continued]
+
+//////////////////////////
+
 All new documents should be indexed via the `mylogs` alias at this point. Adding
 new data to the old indices during the reindexing process can cause data to be
 added to the old indices, but not be reindexed into the new indices.

docs/reference/ilm/set-up-lifecycle-policy.asciidoc

@@ -68,6 +68,16 @@ PUT _template/my_template
 with `test-`
 <2> The template will set the policy to be used to `my_policy`
 
+//////////////////////////
+
+[source,console]
+--------------------------------------------------
+DELETE /_template/my_template
+--------------------------------------------------
+// TEST[continued]
+
+//////////////////////////
+
 Now that a policy exists and is used in an index template we can create an
 initial index which will be managed by our policy:
 

docs/reference/ilm/using-policies-rollover.asciidoc

@@ -101,6 +101,16 @@ PUT _template/my_template
 <2> Associates my_policy with all indices created with this template
 <3> Rolls over the write alias test when the rollover action is triggered
 
+//////////////////////////
+
+[source,console]
+--------------------------------------------------
+DELETE /_template/my_template
+--------------------------------------------------
+// TEST[continued]
+
+//////////////////////////
+
 To be able to start using the policy for these `test-*` indexes we need to
 bootstrap the process by creating the first index.
 

docs/reference/indices/templates.asciidoc

@@ -36,7 +36,7 @@ PUT _template/template_1
 
 [source,console]
 --------------------------------------------------
-DELETE _template/template_1
+DELETE _template/template_*
 --------------------------------------------------
 // TEARDOWN
 
@@ -154,7 +154,6 @@ PUT _template/template_1
     }
 }
 --------------------------------------------------
-// TEST[s/^/DELETE _template\/template_1\n/]
 
 <1> the `{index}` placeholder in the alias name will be replaced with the
 actual index name that the template gets applied to, during index creation.
@@ -195,7 +194,6 @@ PUT /_template/template_2
     }
 }
 --------------------------------------------------
-// TEST[s/^/DELETE _template\/template_1\n/]
 
 The above will disable storing the `_source`, but
 for indices that start with `te*`, `_source` will still be enabled.

docs/reference/mapping/removal_of_types.asciidoc

@@ -645,6 +645,16 @@ PUT index-2-01
 }
 --------------------------------------------------
 
+//////////////////////////
+
+[source,console]
+--------------------------------------------------
+DELETE /_template/template1
+--------------------------------------------------
+// TEST[continued]
+
+//////////////////////////
+
 In case of implicit index creation, because of documents that get indexed in
 an index that doesn't exist yet, the template is always honored. This is
 usually not a problem due to the fact that typeless index calls work on typed

docs/reference/monitoring/indices.asciidoc

@@ -38,6 +38,16 @@ PUT /_template/custom_monitoring
 }
 ----------------------------------
 
+//////////////////////////
+
+[source,console]
+--------------------------------------------------
+DELETE /_template/custom_monitoring
+--------------------------------------------------
+// TEST[continued]
+
+//////////////////////////
+
 IMPORTANT: Only set the `number_of_shards` and `number_of_replicas` in the
 settings section. Overriding other monitoring template settings could cause
 your monitoring dashboards to stop working correctly.

docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java

@@ -24,9 +24,11 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
 
 import org.apache.http.HttpHost;
+import org.apache.http.util.EntityUtils;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.TimeUnits;
 import org.elasticsearch.Version;
+import org.elasticsearch.client.Request;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
@@ -105,6 +107,20 @@ public class DocsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
         if (isMachineLearningTest() || isTransformTest()) {
             ESRestTestCase.waitForPendingTasks(adminClient());
         }
+
+        // check that there are no templates
+        Request request = new Request("GET", "_cat/templates");
+        request.addParameter("h", "name");
+        String templates = EntityUtils.toString(adminClient().performRequest(request).getEntity());
+        if (false == "".equals(templates)) {
+            for (String template : templates.split("\n")) {
+                if (isXPackTemplate(template)) continue;
+                if ("".equals(template)) {
+                    throw new IllegalStateException("empty template in templates list:\n" + templates);
+                }
+                throw new RuntimeException("Template " + template + " not cleared after test");
+            }
+        }
     }
 
     @Override
@@ -117,6 +133,9 @@ public class DocsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
         return isILMTest() == false;
     }
 
+    /**
+     * Tests are themselves responsible for cleaning up templates, which speeds up build.
+     */
     @Override
     protected boolean preserveTemplatesUponCompletion() {
         return true;
@@ -124,7 +143,9 @@ public class DocsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
     protected boolean isSLMTest() {
         String testName = getTestName();
-        return testName != null && (testName.contains("/slm/") || testName.contains("\\slm\\"));
+        return testName != null && (testName.contains("/slm/") || testName.contains("\\slm\\") ||
+            // TODO: Remove after backport of https://github.com/elastic/elasticsearch/pull/48705 which moves SLM docs to correct folder
+            testName.contains("/ilm/") || testName.contains("\\ilm\\"));
     }
 
     protected boolean isILMTest() {

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -1089,7 +1089,7 @@ public abstract class ESRestTestCase extends ESTestCase {
     /**
      * Is this template one that is automatically created by xpack?
      */
-    private static boolean isXPackTemplate(String name) {
+    protected static boolean isXPackTemplate(String name) {
         if (name.startsWith(".monitoring-")) {
             return true;
         }
@@ -1102,10 +1102,14 @@ public abstract class ESRestTestCase extends ESTestCase {
         if (name.startsWith(".ml-")) {
             return true;
         }
+        if (name.startsWith(".transform-")) {
+            return true;
+        }
         switch (name) {
         case ".triggered_watches":
         case ".watches":
         case "logstash-index-template":
+        case ".logstash-management":
         case "security_audit_log":
         case ".slm-history":
             return true;