Validate that snapshot repository exists for ILM policies at creation/update time (#78468)

client/rest-high-level/src/main/java/org/elasticsearch/client/ilm/SearchableSnapshotAction.java

@@ -59,6 +59,10 @@ public class SearchableSnapshotAction implements LifecycleAction, ToXContentObje
         return forceMergeIndex;
     }
 
+    public String getSnapshotRepository() {
+        return snapshotRepository;
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
         builder.startObject();

client/rest-high-level/src/main/java/org/elasticsearch/client/ilm/WaitForSnapshotAction.java

@@ -47,6 +47,10 @@ public class WaitForSnapshotAction implements LifecycleAction, ToXContentObject
         this.policy = policy;
     }
 
+    public String getPolicy() {
+        return policy;
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
         builder.startObject();

client/rest-high-level/src/test/java/org/elasticsearch/client/IndexLifecycleIT.java

@@ -9,6 +9,7 @@
 package org.elasticsearch.client;
 
 import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.elasticsearch.client.core.AcknowledgedResponse;
@@ -40,6 +41,8 @@ import org.elasticsearch.client.ilm.StartILMRequest;
 import org.elasticsearch.client.ilm.StopILMRequest;
 import org.elasticsearch.client.ilm.UnfollowAction;
 import org.elasticsearch.client.ilm.WaitForSnapshotAction;
+import org.elasticsearch.client.slm.PutSnapshotLifecyclePolicyRequest;
+import org.elasticsearch.client.slm.SnapshotLifecyclePolicy;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.TimeValue;
 import org.hamcrest.Matchers;
@@ -50,7 +53,10 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
@@ -65,9 +71,10 @@ public class IndexLifecycleIT extends ESRestHighLevelClientTestCase {
     public void testRemoveIndexLifecyclePolicy() throws Exception {
         String policyName = randomAlphaOfLength(10);
         LifecyclePolicy policy = createRandomPolicy(policyName);
+        ensurePrerequisites(policy);
         PutLifecyclePolicyRequest putRequest = new PutLifecyclePolicyRequest(policy);
         assertAcked(execute(putRequest, highLevelClient().indexLifecycle()::putLifecyclePolicy,
-                highLevelClient().indexLifecycle()::putLifecyclePolicyAsync));
+            highLevelClient().indexLifecycle()::putLifecyclePolicyAsync));
 
         createIndex("foo", Settings.builder().put("index.lifecycle.name", policyName).build());
         createIndex("baz", Settings.builder().put("index.lifecycle.name", policyName).build());
@@ -84,7 +91,7 @@ public class IndexLifecycleIT extends ESRestHighLevelClientTestCase {
         indices.add("rbh");
         RemoveIndexLifecyclePolicyRequest removeReq = new RemoveIndexLifecyclePolicyRequest(indices);
         RemoveIndexLifecyclePolicyResponse removeResp = execute(removeReq, highLevelClient().indexLifecycle()::removeIndexLifecyclePolicy,
-                highLevelClient().indexLifecycle()::removeIndexLifecyclePolicyAsync);
+            highLevelClient().indexLifecycle()::removeIndexLifecyclePolicyAsync);
         assertThat(removeResp.hasFailures(), is(false));
         assertThat(removeResp.getFailedIndexes().isEmpty(), is(true));
 
@@ -98,6 +105,7 @@ public class IndexLifecycleIT extends ESRestHighLevelClientTestCase {
     public void testStartStopILM() throws Exception {
         String policyName = randomAlphaOfLength(10);
         LifecyclePolicy policy = createRandomPolicy(policyName);
+        ensurePrerequisites(policy);
         PutLifecyclePolicyRequest putRequest = new PutLifecyclePolicyRequest(policy);
         assertAcked(execute(putRequest, highLevelClient().indexLifecycle()::putLifecyclePolicy,
             highLevelClient().indexLifecycle()::putLifecyclePolicyAsync));
@@ -115,19 +123,19 @@ public class IndexLifecycleIT extends ESRestHighLevelClientTestCase {
 
         StopILMRequest stopReq = new StopILMRequest();
         AcknowledgedResponse stopResponse = execute(stopReq, highLevelClient().indexLifecycle()::stopILM,
-                highLevelClient().indexLifecycle()::stopILMAsync);
+            highLevelClient().indexLifecycle()::stopILMAsync);
         assertTrue(stopResponse.isAcknowledged());
 
 
         statusResponse = execute(statusRequest, highLevelClient().indexLifecycle()::lifecycleManagementStatus,
             highLevelClient().indexLifecycle()::lifecycleManagementStatusAsync);
         assertThat(statusResponse.getOperationMode(),
-                Matchers.anyOf(equalTo(OperationMode.STOPPING),
-                    equalTo(OperationMode.STOPPED)));
+            Matchers.anyOf(equalTo(OperationMode.STOPPING),
+                equalTo(OperationMode.STOPPED)));
 
         StartILMRequest startReq = new StartILMRequest();
         AcknowledgedResponse startResponse = execute(startReq, highLevelClient().indexLifecycle()::startILM,
-                highLevelClient().indexLifecycle()::startILMAsync);
+            highLevelClient().indexLifecycle()::startILMAsync);
         assertTrue(startResponse.isAcknowledged());
 
         statusResponse = execute(statusRequest, highLevelClient().indexLifecycle()::lifecycleManagementStatus,
@@ -161,6 +169,7 @@ public class IndexLifecycleIT extends ESRestHighLevelClientTestCase {
         lifecyclePhases.put("delete", new Phase("delete", TimeValue.timeValueSeconds(3000), deleteActions));
 
         LifecyclePolicy policy = new LifecyclePolicy(randomAlphaOfLength(10), lifecyclePhases);
+        ensurePrerequisites(policy);
         PutLifecyclePolicyRequest putRequest = new PutLifecyclePolicyRequest(policy);
         AcknowledgedResponse putResponse = execute(putRequest, highLevelClient().indexLifecycle()::putLifecyclePolicy,
             highLevelClient().indexLifecycle()::putLifecyclePolicyAsync);
@@ -215,6 +224,7 @@ public class IndexLifecycleIT extends ESRestHighLevelClientTestCase {
     public void testDeleteLifecycle() throws IOException {
         String policyName = randomAlphaOfLength(10);
         LifecyclePolicy policy = createRandomPolicy(policyName);
+        ensurePrerequisites(policy);
         PutLifecyclePolicyRequest putRequest = new PutLifecyclePolicyRequest(policy);
         assertAcked(execute(putRequest, highLevelClient().indexLifecycle()::putLifecyclePolicy,
             highLevelClient().indexLifecycle()::putLifecyclePolicyAsync));
@@ -233,6 +243,7 @@ public class IndexLifecycleIT extends ESRestHighLevelClientTestCase {
     public void testPutLifecycle() throws IOException {
         String name = randomAlphaOfLengthBetween(5, 20);
         LifecyclePolicy policy = createRandomPolicy(name);
+        ensurePrerequisites(policy);
         PutLifecyclePolicyRequest putRequest = new PutLifecyclePolicyRequest(policy);
 
         assertAcked(execute(putRequest, highLevelClient().indexLifecycle()::putLifecyclePolicy,
@@ -251,6 +262,7 @@ public class IndexLifecycleIT extends ESRestHighLevelClientTestCase {
         for (int i = 0; i < numPolicies; i++) {
             policyNames[i] = "policy-" + randomAlphaOfLengthBetween(5, 10);
             policies[i] = createRandomPolicy(policyNames[i]);
+            ensurePrerequisites(policies[i]);
             PutLifecyclePolicyRequest putRequest = new PutLifecyclePolicyRequest(policies[i]);
             assertAcked(execute(putRequest, highLevelClient().indexLifecycle()::putLifecyclePolicy,
                 highLevelClient().indexLifecycle()::putLifecyclePolicyAsync));
@@ -267,6 +279,7 @@ public class IndexLifecycleIT extends ESRestHighLevelClientTestCase {
     public void testRetryLifecycleStep() throws IOException {
         String policyName = randomAlphaOfLength(10);
         LifecyclePolicy policy = createRandomPolicy(policyName);
+        ensurePrerequisites(policy);
         PutLifecyclePolicyRequest putRequest = new PutLifecyclePolicyRequest(policy);
         assertAcked(execute(putRequest, highLevelClient().indexLifecycle()::putLifecyclePolicy,
             highLevelClient().indexLifecycle()::putLifecyclePolicyAsync));
@@ -285,4 +298,57 @@ public class IndexLifecycleIT extends ESRestHighLevelClientTestCase {
             ex.getRootCause().getMessage()
         );
     }
+
+    public void ensurePrerequisites(LifecyclePolicy policy) throws IOException {
+        Set<String> repositories = policy.getPhases().values().stream()
+            .map(phase -> (SearchableSnapshotAction) phase.getActions().get(SearchableSnapshotAction.NAME))
+            .filter(Objects::nonNull)
+            .map(action -> action.getSnapshotRepository())
+            .collect(Collectors.toSet());
+
+        if (repositories.isEmpty()) {
+            repositories = Set.of("repo");
+        }
+
+        // create any referenced snapshot repositories, or a 'repo' repository if no repositories are otherwise referenced
+        for (String repository : repositories) {
+            createSnapshotRepo(repository, randomBoolean());
+        }
+
+        Set<String> slmPolicies = policy.getPhases().values().stream()
+            .map(phase -> (WaitForSnapshotAction) phase.getActions().get(WaitForSnapshotAction.NAME))
+            .filter(Objects::nonNull)
+            .map(action -> action.getPolicy())
+            .collect(Collectors.toSet());
+
+        // create any slm policies, using one of the snapshot repositories previously created
+        for (String slmPolicy : slmPolicies) {
+            createSlmPolicy(slmPolicy, randomFrom(repositories));
+        }
+    }
+
+    public static void createSnapshotRepo(String repoName, boolean compress) throws IOException {
+        PutRepositoryRequest request = new PutRepositoryRequest(repoName)
+            .type("fs")
+            .settings(Settings.builder()
+                .put("compress", compress)
+                .put("location", System.getProperty("tests.path.repo") + "/" + randomAlphaOfLengthBetween(4, 10))
+                .put("max_snapshot_bytes_per_sec", "100m"));
+        assertTrue(highLevelClient().snapshot()
+            .createRepository(request, RequestOptions.DEFAULT)
+            .isAcknowledged());
+    }
+
+    private void createSlmPolicy(String slmPolicy, String repo) throws IOException {
+        PutSnapshotLifecyclePolicyRequest request = new PutSnapshotLifecyclePolicyRequest(new SnapshotLifecyclePolicy(
+            slmPolicy,
+            "snap" + randomAlphaOfLengthBetween(5, 10).toLowerCase(Locale.ROOT),
+            "59 59 23 31 12 ? 2099",
+            repo,
+            null,
+            null));
+        assertTrue(highLevelClient().indexLifecycle().
+            putSnapshotLifecyclePolicy(request, RequestOptions.DEFAULT)
+            .isAcknowledged());
+    }
 }

docs/reference/data-streams/set-up-a-data-stream.asciidoc

@@ -29,6 +29,20 @@ To create an index lifecycle policy in {kib}, open the main menu and go to
 
 You can also use the <<ilm-put-lifecycle,create lifecycle policy API>>.
 
+////
+[source,console]
+--------------------------------------------------
+PUT /_snapshot/found-snapshots
+{
+ "type": "fs",
+  "settings": {
+    "location": "my_backup_location"
+  }
+}
+--------------------------------------------------
+// TESTSETUP
+////
+
 // tag::ilm-policy-api-ex[]
 [source,console]
 ----

docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc

@@ -61,6 +61,19 @@ are force merged.
 
 [[ilm-searchable-snapshot-ex]]
 ==== Examples
+////
+[source,console]
+--------------------------------------------------
+PUT /_snapshot/backing_repo
+{
+ "type": "fs",
+  "settings": {
+    "location": "my_backup_location"
+  }
+}
+--------------------------------------------------
+// TESTSETUP
+////
 [source,console]
 --------------------------------------------------
 PUT _ilm/policy/my_policy

docs/reference/ilm/actions/ilm-wait-for-snapshot.asciidoc

@@ -5,7 +5,7 @@
 Phases allowed: delete.
 
 Waits for the specified {slm-init} policy to be executed before removing the index.
-This ensures that a snapshot of the deleted index is available. 
+This ensures that a snapshot of the deleted index is available.
 
 [[ilm-wait-for-snapshot-options]]
 ==== Options
@@ -16,7 +16,26 @@ Name of the {slm-init} policy that the delete action should wait for.
 
 [[ilm-wait-for-snapshot-ex]]
 ==== Example
+////
+[source,console]
+--------------------------------------------------
+PUT /_snapshot/backing_repo
+{
+ "type": "fs",
+  "settings": {
+    "location": "my_backup_location"
+  }
+}
 
+PUT /_slm/policy/slm-policy-name
+{
+  "schedule": "0 30 1 * * ?",
+  "name": "<daily-snap-{now/d}>",
+  "repository": "backing_repo"
+}
+--------------------------------------------------
+// TESTSETUP
+////
 [source,console]
 --------------------------------------------------
 PUT _ilm/policy/my_policy
@@ -33,4 +52,4 @@ PUT _ilm/policy/my_policy
     }
   }
 }
---------------------------------------------------
+--------------------------------------------------

docs/reference/migration/migrate_8_0/ilm.asciidoc

@@ -50,4 +50,18 @@ has been removed in 8.0.
 *Impact* +
 Update your ILM policies to remove the `freeze` action from the `cold` phase.
 ====
+
+[[ilm-policy-validation]]
+.Additional validation for ILM policies.
+[%collapsible]
+====
+*Details* +
+Creating or updating an ILM policy now requires that any referenced snapshot repositories and SLM
+policies exist.
+
+*Impact* +
+Update your code or configuration management to ensure that repositories and SLM policies are created
+before any policies that reference them.
+====
+
 // end::notable-breaking-changes[]

docs/reference/tab-widgets/ilm.asciidoc

@@ -13,6 +13,20 @@ Index Lifecycle Policies**. Click the policy you'd like to edit.
 
 You can also use the <<ilm-put-lifecycle,update lifecycle policy API>>.
 
+////
+[source,console]
+--------------------------------------------------
+PUT /_snapshot/found-snapshots
+{
+ "type": "fs",
+  "settings": {
+    "location": "my_backup_location"
+  }
+}
+--------------------------------------------------
+// TESTSETUP
+////
+
 [source,console]
 ----
 PUT _ilm/policy/logs

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotAction.java

@@ -53,6 +53,10 @@ public class WaitForSnapshotAction implements LifecycleAction {
         this(in.readString());
     }
 
+    public String getPolicy() {
+        return policy;
+    }
+
     @Override
     public List<Step> toSteps(Client client, String phase, StepKey nextStepKey) {
         StepKey waitForSnapshotKey = new StepKey(phase, NAME, WaitForSnapshotStep.NAME);

x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java

@@ -190,8 +190,13 @@ public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
         createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
             .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0));
         String slmPolicy = randomAlphaOfLengthBetween(4, 10);
+        String snapshotRepo = randomAlphaOfLengthBetween(4, 10);
+        createSnapshotRepo(client(), snapshotRepo, randomBoolean());
+        createSlmPolicy(slmPolicy, snapshotRepo);
+
         final String phaseName = "delete";
         createNewSingletonPolicy(client(), policy, phaseName, new WaitForSnapshotAction(slmPolicy));
+        deleteSlmPolicy(slmPolicy); // delete the slm policy out from underneath ilm
         updatePolicy(client(), index, policy);
         waitForPhaseTime(phaseName);
         assertBusy(() -> {
@@ -199,9 +204,7 @@ public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
             assertThat(indexILMState.get("action"), is("wait_for_snapshot"));
             assertThat(indexILMState.get("failed_step"), is("wait-for-snapshot"));
         }, slmPolicy);
-        String snapshotRepo = randomAlphaOfLengthBetween(4, 10);
-        createSnapshotRepo(client(), snapshotRepo, randomBoolean());
-        createSlmPolicy(slmPolicy, snapshotRepo);
+        createSlmPolicy(slmPolicy, snapshotRepo); // put the slm policy back
         assertBusy(() -> {
             Map<String, Object> indexILMState = explainIndex(client(), index);
             //wait for step to notice that the slm policy is created and to get out of error
@@ -228,10 +231,11 @@ public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
         createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
             .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0));
         String slmPolicy = randomAlphaOfLengthBetween(4, 10);
-        final String phaseName = "delete";
         String snapshotRepo = randomAlphaOfLengthBetween(4, 10);
         createSnapshotRepo(client(), snapshotRepo, randomBoolean());
         createSlmPolicy(slmPolicy, snapshotRepo);
+
+        final String phaseName = "delete";
         createNewSingletonPolicy(client(), policy, phaseName, new WaitForSnapshotAction(slmPolicy));
         updatePolicy(client(), index, policy);
         waitForPhaseTime(phaseName);
@@ -249,13 +253,13 @@ public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
         createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
             .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0));
         String slmPolicy = randomAlphaOfLengthBetween(4, 10);
-        final String phaseName = "delete";
-        createNewSingletonPolicy(client(), policy, phaseName, new WaitForSnapshotAction(slmPolicy));
-
         String snapshotRepo = randomAlphaOfLengthBetween(4, 10);
         createSnapshotRepo(client(), snapshotRepo, randomBoolean());
         createSlmPolicy(slmPolicy, snapshotRepo);
 
+        final String phaseName = "delete";
+        createNewSingletonPolicy(client(), policy, phaseName, new WaitForSnapshotAction(slmPolicy));
+
         Request request = new Request("PUT", "/_slm/policy/" + slmPolicy + "/_execute");
         assertOK(client().performRequest(request));
 
@@ -435,7 +439,7 @@ public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
 
     @SuppressWarnings("unchecked")
     public void testNonexistentPolicy() throws Exception {
-        String indexPrefix = randomAlphaOfLengthBetween(5,15).toLowerCase(Locale.ROOT);
+        String indexPrefix = randomAlphaOfLengthBetween(5, 15).toLowerCase(Locale.ROOT);
         final StringEntity template = new StringEntity("{\n" +
             "  \"index_patterns\": \"" + indexPrefix + "*\",\n" +
             "  \"settings\": {\n" +
@@ -452,7 +456,7 @@ public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
         templateRequest.setOptions(expectWarnings(RestPutIndexTemplateAction.DEPRECATION_WARNING));
         client().performRequest(templateRequest);
 
-        policy = randomAlphaOfLengthBetween(5,20);
+        policy = randomAlphaOfLengthBetween(5, 20);
         createNewSingletonPolicy(client(), policy, "hot", new RolloverAction(null, null, null, 1L));
 
         index = indexPrefix + "-000001";
@@ -476,7 +480,7 @@ public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
                 responseMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), is, true);
             }
             logger.info(responseMap);
-            Map<String, Object> indexStatus = (Map<String, Object>)((Map<String, Object>) responseMap.get("indices")).get(index);
+            Map<String, Object> indexStatus = (Map<String, Object>) ((Map<String, Object>) responseMap.get("indices")).get(index);
             assertNull(indexStatus.get("phase"));
             assertNull(indexStatus.get("action"));
             assertNull(indexStatus.get("step"));
@@ -490,11 +494,11 @@ public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
     public void testInvalidPolicyNames() {
         ResponseException ex;
 
-        policy = randomAlphaOfLengthBetween(0,10) + "," + randomAlphaOfLengthBetween(0,10);
+        policy = randomAlphaOfLengthBetween(0, 10) + "," + randomAlphaOfLengthBetween(0, 10);
         ex = expectThrows(ResponseException.class, () -> createNewSingletonPolicy(client(), policy, "delete", new DeleteAction()));
         assertThat(ex.getMessage(), containsString("invalid policy name"));
 
-        policy = randomAlphaOfLengthBetween(0,10) + "%20" + randomAlphaOfLengthBetween(0,10);
+        policy = randomAlphaOfLengthBetween(0, 10) + "%20" + randomAlphaOfLengthBetween(0, 10);
         ex = expectThrows(ResponseException.class, () -> createNewSingletonPolicy(client(), policy, "delete", new DeleteAction()));
         assertThat(ex.getMessage(), containsString("invalid policy name"));
 
@@ -585,7 +589,7 @@ public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
     public void testCanStopILMWithPolicyUsingNonexistentPolicy() throws Exception {
         createIndexWithSettings(client(), index, alias, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
             .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
-            .put(LifecycleSettings.LIFECYCLE_NAME_SETTING.getKey(), randomAlphaOfLengthBetween(5,15)));
+            .put(LifecycleSettings.LIFECYCLE_NAME_SETTING.getKey(), randomAlphaOfLengthBetween(5, 15)));
 
         Request stopILMRequest = new Request("POST", "_ilm/stop");
         assertOK(client().performRequest(stopILMRequest));
@@ -618,7 +622,7 @@ public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
         updatePolicy(client(), originalIndex, policy);
         Request createIndexTemplate = new Request("PUT", "_template/rolling_indexes");
         createIndexTemplate.setJsonEntity("{" +
-            "\"index_patterns\": [\""+ index + "-*\"], \n" +
+            "\"index_patterns\": [\"" + index + "-*\"], \n" +
             "  \"settings\": {\n" +
             "    \"number_of_shards\": 1,\n" +
             "    \"number_of_replicas\": 142,\n" +
@@ -644,11 +648,11 @@ public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
         createNewSingletonPolicy(client(), policy, "hot", new RolloverAction(null, null, null, 1L));
         Request createIndexTemplate = new Request("PUT", "_template/rolling_indexes");
         createIndexTemplate.setJsonEntity("{" +
-            "\"index_patterns\": [\""+ index + "-*\"], \n" +
+            "\"index_patterns\": [\"" + index + "-*\"], \n" +
             "  \"settings\": {\n" +
             "    \"number_of_shards\": 1,\n" +
             "    \"number_of_replicas\": 0,\n" +
-            "    \"index.lifecycle.name\": \"" + policy+ "\",\n" +
+            "    \"index.lifecycle.name\": \"" + policy + "\",\n" +
             "    \"index.lifecycle.rollover_alias\": \"" + alias + "\"\n" +
             "  }\n" +
             "}");
@@ -748,11 +752,11 @@ public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
         createNewSingletonPolicy(client(), policy, "hot", new RolloverAction(null, null, null, 100L));
         Request createIndexTemplate = new Request("PUT", "_template/rolling_indexes");
         createIndexTemplate.setJsonEntity("{" +
-            "\"index_patterns\": [\""+ index + "-*\"], \n" +
+            "\"index_patterns\": [\"" + index + "-*\"], \n" +
             "  \"settings\": {\n" +
             "    \"number_of_shards\": 1,\n" +
             "    \"number_of_replicas\": 0,\n" +
-            "    \"index.lifecycle.name\": \"" + policy+ "\",\n" +
+            "    \"index.lifecycle.name\": \"" + policy + "\",\n" +
             "    \"index.lifecycle.rollover_alias\": \"" + alias + "\"\n" +
             "  }\n" +
             "}");
@@ -883,8 +887,8 @@ public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
                     for (Object snapshot : snapshots) {
                         Map<String, Object> snapshotInfoMap = (Map<String, Object>) snapshot;
                         if (snapshotInfoMap.get("snapshot").equals(snapshotName[0]) &&
-                                // wait for the snapshot to be completed (successfully or not) otherwise the teardown might fail
-                                SnapshotState.valueOf((String) snapshotInfoMap.get("state")).completed()) {
+                            // wait for the snapshot to be completed (successfully or not) otherwise the teardown might fail
+                            SnapshotState.valueOf((String) snapshotInfoMap.get("state")).completed()) {
                             return true;
                         }
                     }
@@ -896,6 +900,26 @@ public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
         }, 30, TimeUnit.SECONDS));
     }
 
+    public void testSearchableSnapshotRequiresSnapshotRepoToExist() throws IOException {
+        String repo = randomAlphaOfLengthBetween(4, 10);
+        final String phaseName = "cold";
+        ResponseException ex = expectThrows(ResponseException.class, () ->
+            createNewSingletonPolicy(client(), policy, phaseName, new SearchableSnapshotAction(repo)));
+        assertThat(ex.getMessage(), containsString("no such repository"));
+        assertThat(ex.getMessage(), containsString("the snapshot repository referenced by the [searchable_snapshot] action " +
+            "in the [cold] phase must exist before it can be referenced by an ILM policy"));
+    }
+
+    public void testWaitForSnapshotRequiresSLMPolicyToExist() throws IOException {
+        String slmPolicy = randomAlphaOfLengthBetween(4, 10);
+        final String phaseName = "delete";
+        ResponseException ex = expectThrows(ResponseException.class, () ->
+            createNewSingletonPolicy(client(), policy, phaseName, new WaitForSnapshotAction(slmPolicy)));
+        assertThat(ex.getMessage(), containsString("no such snapshot lifecycle policy"));
+        assertThat(ex.getMessage(), containsString("the snapshot lifecycle policy referenced by the [wait_for_snapshot] action " +
+            "in the [delete] phase must exist before it can be referenced by an ILM policy"));
+    }
+
     // This method should be called inside an assertBusy, it has no retry logic of its own
     private void assertHistoryIsPresent(String policyName, String indexName, boolean success, String stepName) throws IOException {
         assertHistoryIsPresent(policyName, indexName, success, null, null, stepName);
@@ -946,7 +970,7 @@ public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
                 historyResponseMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), is, true);
             }
             logger.info("--> history response: {}", historyResponseMap);
-            int hits = (int)((Map<String, Object>) ((Map<String, Object>) historyResponseMap.get("hits")).get("total")).get("value");
+            int hits = (int) ((Map<String, Object>) ((Map<String, Object>) historyResponseMap.get("hits")).get("total")).get("value");
 
             // For a failure, print out whatever history we *do* have for the index
             if (hits == 0) {
@@ -1007,6 +1031,10 @@ public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
         assertOK(client().performRequest(request));
     }
 
+    private void deleteSlmPolicy(String smlPolicy) throws IOException {
+        assertOK(client().performRequest(new Request("DELETE", "/_slm/policy/" + smlPolicy)));
+    }
+
     //adds debug information for waitForSnapshot tests
     private void assertBusy(CheckedRunnable<Exception> runnable, String slmPolicy) throws Exception {
         assertBusy(() -> {

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleAction.java

@@ -21,6 +21,7 @@ import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.metadata.RepositoriesMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
@@ -34,8 +35,10 @@ import org.elasticsearch.xpack.core.ilm.LifecyclePolicy;
 import org.elasticsearch.xpack.core.ilm.LifecyclePolicyMetadata;
 import org.elasticsearch.xpack.core.ilm.Phase;
 import org.elasticsearch.xpack.core.ilm.SearchableSnapshotAction;
+import org.elasticsearch.xpack.core.ilm.WaitForSnapshotAction;
 import org.elasticsearch.xpack.core.ilm.action.PutLifecycleAction;
 import org.elasticsearch.xpack.core.ilm.action.PutLifecycleAction.Request;
+import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata;
 
 import java.time.Instant;
 import java.util.List;
@@ -76,56 +79,96 @@ public class TransportPutLifecycleAction extends TransportMasterNodeAction<Reque
         // cluster state thread in the ClusterStateUpdateTask below since that thread does not share the
         // same context, and therefore does not have access to the appropriate security headers.
         Map<String, String> filteredHeaders = ClientHelper.filterSecurityHeaders(threadPool.getThreadContext().getHeaders());
+
         LifecyclePolicy.validatePolicyName(request.getPolicy().getName());
-        List<Phase> phasesDefiningSearchableSnapshot = request.getPolicy().getPhases().values().stream()
-            .filter(phase -> phase.getActions().containsKey(SearchableSnapshotAction.NAME))
-            .collect(Collectors.toList());
-        if (phasesDefiningSearchableSnapshot.isEmpty() == false) {
-            if (SEARCHABLE_SNAPSHOT_FEATURE.checkWithoutTracking(licenseState) == false) {
-                throw new IllegalArgumentException("policy [" + request.getPolicy().getName() + "] defines the [" +
-                    SearchableSnapshotAction.NAME + "] action but the current license is non-compliant for [searchable-snapshots]");
-            }
-        }
+
         clusterService.submitStateUpdateTask("put-lifecycle-" + request.getPolicy().getName(),
-                new AckedClusterStateUpdateTask(request, listener) {
-                    @Override
-                    public ClusterState execute(ClusterState currentState) throws Exception {
-                        ClusterState.Builder stateBuilder = ClusterState.builder(currentState);
-                        IndexLifecycleMetadata currentMetadata = currentState.metadata().custom(IndexLifecycleMetadata.TYPE);
-                        if (currentMetadata == null) { // first time using index-lifecycle feature, bootstrap metadata
-                            currentMetadata = IndexLifecycleMetadata.EMPTY;
-                        }
-                        LifecyclePolicyMetadata existingPolicyMetadata = currentMetadata.getPolicyMetadatas()
-                            .get(request.getPolicy().getName());
-                        long nextVersion = (existingPolicyMetadata == null) ? 1L : existingPolicyMetadata.getVersion() + 1L;
-                        SortedMap<String, LifecyclePolicyMetadata> newPolicies = new TreeMap<>(currentMetadata.getPolicyMetadatas());
-                        LifecyclePolicyMetadata lifecyclePolicyMetadata = new LifecyclePolicyMetadata(request.getPolicy(), filteredHeaders,
-                            nextVersion, Instant.now().toEpochMilli());
-                        LifecyclePolicyMetadata oldPolicy = newPolicies.put(lifecyclePolicyMetadata.getName(), lifecyclePolicyMetadata);
-                        if (oldPolicy == null) {
-                            logger.info("adding index lifecycle policy [{}]", request.getPolicy().getName());
-                        } else {
-                            logger.info("updating index lifecycle policy [{}]", request.getPolicy().getName());
-                        }
-                        IndexLifecycleMetadata newMetadata = new IndexLifecycleMetadata(newPolicies, currentMetadata.getOperationMode());
-                        stateBuilder.metadata(Metadata.builder(currentState.getMetadata())
-                                .putCustom(IndexLifecycleMetadata.TYPE, newMetadata).build());
-                        ClusterState nonRefreshedState = stateBuilder.build();
-                        if (oldPolicy == null) {
+            new AckedClusterStateUpdateTask(request, listener) {
+                @Override
+                public ClusterState execute(ClusterState currentState) throws Exception {
+                    validatePrerequisites(request.getPolicy(), currentState);
+
+                    ClusterState.Builder stateBuilder = ClusterState.builder(currentState);
+                    IndexLifecycleMetadata currentMetadata = currentState.metadata().custom(IndexLifecycleMetadata.TYPE);
+                    if (currentMetadata == null) { // first time using index-lifecycle feature, bootstrap metadata
+                        currentMetadata = IndexLifecycleMetadata.EMPTY;
+                    }
+                    LifecyclePolicyMetadata existingPolicyMetadata = currentMetadata.getPolicyMetadatas()
+                        .get(request.getPolicy().getName());
+                    long nextVersion = (existingPolicyMetadata == null) ? 1L : existingPolicyMetadata.getVersion() + 1L;
+                    SortedMap<String, LifecyclePolicyMetadata> newPolicies = new TreeMap<>(currentMetadata.getPolicyMetadatas());
+                    LifecyclePolicyMetadata lifecyclePolicyMetadata = new LifecyclePolicyMetadata(request.getPolicy(), filteredHeaders,
+                        nextVersion, Instant.now().toEpochMilli());
+                    LifecyclePolicyMetadata oldPolicy = newPolicies.put(lifecyclePolicyMetadata.getName(), lifecyclePolicyMetadata);
+                    if (oldPolicy == null) {
+                        logger.info("adding index lifecycle policy [{}]", request.getPolicy().getName());
+                    } else {
+                        logger.info("updating index lifecycle policy [{}]", request.getPolicy().getName());
+                    }
+                    IndexLifecycleMetadata newMetadata = new IndexLifecycleMetadata(newPolicies, currentMetadata.getOperationMode());
+                    stateBuilder.metadata(Metadata.builder(currentState.getMetadata())
+                        .putCustom(IndexLifecycleMetadata.TYPE, newMetadata).build());
+                    ClusterState nonRefreshedState = stateBuilder.build();
+                    if (oldPolicy == null) {
+                        return nonRefreshedState;
+                    } else {
+                        try {
+                            return updateIndicesForPolicy(nonRefreshedState, xContentRegistry, client,
+                                oldPolicy.getPolicy(), lifecyclePolicyMetadata, licenseState);
+                        } catch (Exception e) {
+                            logger.warn(new ParameterizedMessage("unable to refresh indices phase JSON for updated policy [{}]",
+                                oldPolicy.getName()), e);
+                            // Revert to the non-refreshed state
                             return nonRefreshedState;
-                        } else {
-                            try {
-                                return updateIndicesForPolicy(nonRefreshedState, xContentRegistry, client,
-                                    oldPolicy.getPolicy(), lifecyclePolicyMetadata, licenseState);
-                            } catch (Exception e) {
-                                logger.warn(new ParameterizedMessage("unable to refresh indices phase JSON for updated policy [{}]",
-                                    oldPolicy.getName()), e);
-                                // Revert to the non-refreshed state
-                                return nonRefreshedState;
-                            }
                         }
                     }
-                });
+                }
+            });
+    }
+
+    /**
+     * Validate that the license level is compliant for searchable-snapshots, that any referenced snapshot
+     * repositories exist, and that any referenced SLM policies exist.
+     *
+     * @param policy The lifecycle policy
+     * @param state The cluster state
+     */
+    private void validatePrerequisites(LifecyclePolicy policy, ClusterState state) {
+        List<Phase> phasesWithSearchableSnapshotActions = policy.getPhases().values().stream()
+            .filter(phase -> phase.getActions().containsKey(SearchableSnapshotAction.NAME))
+            .collect(Collectors.toList());
+        // check license level for searchable snapshots
+        if (phasesWithSearchableSnapshotActions.isEmpty() == false &&
+            SEARCHABLE_SNAPSHOT_FEATURE.checkWithoutTracking(licenseState) == false) {
+            throw new IllegalArgumentException("policy [" + policy.getName() + "] defines the [" +
+                SearchableSnapshotAction.NAME + "] action but the current license is non-compliant for [searchable-snapshots]");
+        }
+        // make sure any referenced snapshot repositories exist
+        for (Phase phase : phasesWithSearchableSnapshotActions) {
+            SearchableSnapshotAction action = (SearchableSnapshotAction) phase.getActions().get(SearchableSnapshotAction.NAME);
+            String repository = action.getSnapshotRepository();
+            if (state.metadata().custom(RepositoriesMetadata.TYPE, RepositoriesMetadata.EMPTY)
+                .repository(repository) == null) {
+                throw new IllegalArgumentException("no such repository [" + repository + "], the snapshot repository " +
+                    "referenced by the [" + SearchableSnapshotAction.NAME + "] action in the [" + phase.getName() + "] phase " +
+                    "must exist before it can be referenced by an ILM policy");
+            }
+        }
+
+        List<Phase> phasesWithWaitForSnapshotActions = policy.getPhases().values().stream()
+            .filter(phase -> phase.getActions().containsKey(WaitForSnapshotAction.NAME))
+            .collect(Collectors.toList());
+        // make sure any referenced snapshot lifecycle policies exist
+        for (Phase phase : phasesWithWaitForSnapshotActions) {
+            WaitForSnapshotAction action = (WaitForSnapshotAction) phase.getActions().get(WaitForSnapshotAction.NAME);
+            String slmPolicy = action.getPolicy();
+            if (state.metadata().custom(SnapshotLifecycleMetadata.TYPE, SnapshotLifecycleMetadata.EMPTY)
+                .getSnapshotConfigurations().get(slmPolicy) == null) {
+                throw new IllegalArgumentException("no such snapshot lifecycle policy [" + slmPolicy + "], the snapshot lifecycle policy " +
+                    "referenced by the [" + WaitForSnapshotAction.NAME + "] action in the [" + phase.getName() + "] phase " +
+                    "must exist before it can be referenced by an ILM policy");
+            }
+        }
     }
 
     @Override

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/TransportExecuteSnapshotLifecycleAction.java

@@ -53,12 +53,7 @@ public class TransportExecuteSnapshotLifecycleAction
                                    final ActionListener<ExecuteSnapshotLifecycleAction.Response> listener) {
         try {
             final String policyId = request.getLifecycleId();
-            SnapshotLifecycleMetadata snapMeta = state.metadata().custom(SnapshotLifecycleMetadata.TYPE);
-            if (snapMeta == null) {
-                listener.onFailure(new IllegalArgumentException("no such snapshot lifecycle policy [" + policyId + "]"));
-                return;
-            }
-
+            SnapshotLifecycleMetadata snapMeta = state.metadata().custom(SnapshotLifecycleMetadata.TYPE, SnapshotLifecycleMetadata.EMPTY);
             SnapshotLifecyclePolicyMetadata policyMetadata = snapMeta.getSnapshotConfigurations().get(policyId);
             if (policyMetadata == null) {
                 listener.onFailure(new IllegalArgumentException("no such snapshot lifecycle policy [" + policyId + "]"));