Fix ExplainLifecycleIT.testExplainFilters (#74941)

Explain `onlyErrors` is tricky to integration test as all the ILM steps
are now retryable (so the `explain` API might not catch a particular
failing step when it's in the `ERROR` state as when we retry we move it
back into the step that failed).

This adds an unit test for the "explain onlyErrors" case.

x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/ExplainLifecycleIT.java

@@ -15,8 +15,8 @@ import org.elasticsearch.client.Request;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.xpack.core.ilm.DeleteAction;
 import org.elasticsearch.xpack.core.ilm.LifecycleAction;
@@ -61,7 +61,6 @@ public class ExplainLifecycleIT extends ESRestTestCase {
         alias = "alias-" + randomAlphaOfLength(5);
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/71010")
     public void testExplainFilters() throws Exception {
         String goodIndex = index + "-good-000001";
         String errorIndex = index + "-error";
@@ -115,7 +114,7 @@ public class ExplainLifecycleIT extends ESRestTestCase {
 
             Map<String, Map<String, Object>> onlyErrorsResponse = explain(client(), index + "*", true, true);
             assertNotNull(onlyErrorsResponse);
-            assertThat(onlyErrorsResponse, allOf(hasKey(errorIndex), hasKey(nonexistantPolicyIndex)));
+            assertThat(onlyErrorsResponse, hasKey(nonexistantPolicyIndex));
             assertThat(onlyErrorsResponse, allOf(not(hasKey(goodIndex)), not(hasKey(unmanagedIndex))));
         });
     }

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportExplainLifecycleAction.java

@@ -23,6 +23,7 @@ import org.elasticsearch.common.xcontent.DeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -52,8 +53,8 @@ public class TransportExplainLifecycleAction
     public TransportExplainLifecycleAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool,
                                            ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
                                            NamedXContentRegistry xContentRegistry, IndexLifecycleService indexLifecycleService) {
-        super(ExplainLifecycleAction.NAME, transportService, clusterService, threadPool, actionFilters,
-                ExplainLifecycleRequest::new, indexNameExpressionResolver, ExplainLifecycleResponse::new);
+        super(ExplainLifecycleAction.NAME, transportService, clusterService, threadPool, actionFilters, ExplainLifecycleRequest::new,
+            indexNameExpressionResolver, ExplainLifecycleResponse::new);
         this.xContentRegistry = xContentRegistry;
         this.indexLifecycleService = indexLifecycleService;
     }
@@ -64,57 +65,13 @@ public class TransportExplainLifecycleAction
         Map<String, IndexLifecycleExplainResponse> indexResponses = new HashMap<>();
         for (String index : concreteIndices) {
             IndexMetadata idxMetadata = state.metadata().index(index);
-            Settings idxSettings = idxMetadata.getSettings();
-            LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(idxMetadata);
-            String policyName = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(idxSettings);
-            String currentPhase = lifecycleState.getPhase();
-            String stepInfo = lifecycleState.getStepInfo();
-            BytesArray stepInfoBytes = null;
-            if (stepInfo != null) {
-                stepInfoBytes = new BytesArray(stepInfo);
-            }
-            // parse existing phase steps from the phase definition in the index settings
-            String phaseDef = lifecycleState.getPhaseDefinition();
-            PhaseExecutionInfo phaseExecutionInfo = null;
-            if (Strings.isNullOrEmpty(phaseDef) == false) {
-                try (XContentParser parser = JsonXContent.jsonXContent.createParser(xContentRegistry,
-                        DeprecationHandler.THROW_UNSUPPORTED_OPERATION, phaseDef)) {
-                    phaseExecutionInfo = PhaseExecutionInfo.parse(parser, currentPhase);
-                } catch (IOException e) {
-                    listener.onFailure(new ElasticsearchParseException(
-                        "failed to parse phase definition for index [" + index + "]", e));
-                    return;
-                }
-            }
             final IndexLifecycleExplainResponse indexResponse;
-            if (Strings.hasLength(policyName)) {
-                // If this is requesting only errors, only include indices in the error step or which are using a nonexistent policy
-                if (request.onlyErrors() == false
-                    || (ErrorStep.NAME.equals(lifecycleState.getStep()) || indexLifecycleService.policyExists(policyName) == false)) {
-                    Long originationDate = idxSettings.getAsLong(LIFECYCLE_ORIGINATION_DATE, -1L);
-                    indexResponse = IndexLifecycleExplainResponse.newManagedIndexResponse(index, policyName,
-                        originationDate != -1L ? originationDate : lifecycleState.getLifecycleDate(),
-                        lifecycleState.getPhase(),
-                        lifecycleState.getAction(),
-                        lifecycleState.getStep(),
-                        lifecycleState.getFailedStep(),
-                        lifecycleState.isAutoRetryableError(),
-                        lifecycleState.getFailedStepRetryCount(),
-                        lifecycleState.getPhaseTime(),
-                        lifecycleState.getActionTime(),
-                        lifecycleState.getStepTime(),
-                        lifecycleState.getSnapshotRepository(),
-                        lifecycleState.getSnapshotName(),
-                        lifecycleState.getShrinkIndexName(),
-                        stepInfoBytes,
-                        phaseExecutionInfo);
-                } else {
-                    indexResponse = null;
-                }
-            } else if (request.onlyManaged() == false && request.onlyErrors() == false) {
-                indexResponse = IndexLifecycleExplainResponse.newUnmanagedIndexResponse(index);
-            } else {
-                indexResponse = null;
+            try {
+                indexResponse = getIndexLifecycleExplainResponse(idxMetadata, request.onlyErrors(), request.onlyManaged(),
+                    indexLifecycleService, xContentRegistry);
+            } catch (IOException e) {
+                listener.onFailure(new ElasticsearchParseException("failed to parse phase definition for index [" + index + "]", e));
+                return;
             }
 
             if (indexResponse != null) {
@@ -124,4 +81,61 @@ public class TransportExplainLifecycleAction
         listener.onResponse(new ExplainLifecycleResponse(indexResponses));
     }
 
+    @Nullable
+    static IndexLifecycleExplainResponse getIndexLifecycleExplainResponse(IndexMetadata indexMetadata, boolean onlyErrors,
+                                                                          boolean onlyManaged, IndexLifecycleService indexLifecycleService,
+                                                                          NamedXContentRegistry xContentRegistry) throws IOException {
+        Settings idxSettings = indexMetadata.getSettings();
+        LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(indexMetadata);
+        String policyName = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(idxSettings);
+        String currentPhase = lifecycleState.getPhase();
+        String stepInfo = lifecycleState.getStepInfo();
+        BytesArray stepInfoBytes = null;
+        if (stepInfo != null) {
+            stepInfoBytes = new BytesArray(stepInfo);
+        }
+        String indexName = indexMetadata.getIndex().getName();
+
+        // parse existing phase steps from the phase definition in the index settings
+        String phaseDef = lifecycleState.getPhaseDefinition();
+        PhaseExecutionInfo phaseExecutionInfo = null;
+        if (Strings.isNullOrEmpty(phaseDef) == false) {
+            try (XContentParser parser = JsonXContent.jsonXContent.createParser(xContentRegistry,
+                DeprecationHandler.THROW_UNSUPPORTED_OPERATION, phaseDef)) {
+                phaseExecutionInfo = PhaseExecutionInfo.parse(parser, currentPhase);
+            }
+        }
+
+        final IndexLifecycleExplainResponse indexResponse;
+        if (Strings.hasLength(policyName)) {
+            // If this is requesting only errors, only include indices in the error step or which are using a nonexistent policy
+            if (onlyErrors == false
+                || (ErrorStep.NAME.equals(lifecycleState.getStep()) || indexLifecycleService.policyExists(policyName) == false)) {
+                Long originationDate = idxSettings.getAsLong(LIFECYCLE_ORIGINATION_DATE, -1L);
+                indexResponse = IndexLifecycleExplainResponse.newManagedIndexResponse(indexName, policyName,
+                    originationDate != -1L ? originationDate : lifecycleState.getLifecycleDate(),
+                    lifecycleState.getPhase(),
+                    lifecycleState.getAction(),
+                    lifecycleState.getStep(),
+                    lifecycleState.getFailedStep(),
+                    lifecycleState.isAutoRetryableError(),
+                    lifecycleState.getFailedStepRetryCount(),
+                    lifecycleState.getPhaseTime(),
+                    lifecycleState.getActionTime(),
+                    lifecycleState.getStepTime(),
+                    lifecycleState.getSnapshotRepository(),
+                    lifecycleState.getSnapshotName(),
+                    lifecycleState.getShrinkIndexName(),
+                    stepInfoBytes,
+                    phaseExecutionInfo);
+            } else {
+                indexResponse = null;
+            }
+        } else if (onlyManaged == false && onlyErrors == false) {
+            indexResponse = IndexLifecycleExplainResponse.newUnmanagedIndexResponse(indexName);
+        } else {
+            indexResponse = null;
+        }
+        return indexResponse;
+    }
 }

x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportExplainLifecycleActionTests.java

@@ -0,0 +1,150 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.ilm.action;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ParseField;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.ilm.ErrorStep;
+import org.elasticsearch.xpack.core.ilm.IndexLifecycleExplainResponse;
+import org.elasticsearch.xpack.core.ilm.LifecycleAction;
+import org.elasticsearch.xpack.core.ilm.LifecycleExecutionState;
+import org.elasticsearch.xpack.core.ilm.LifecycleSettings;
+import org.elasticsearch.xpack.core.ilm.RolloverAction;
+import org.elasticsearch.xpack.core.ilm.WaitForRolloverReadyStep;
+import org.elasticsearch.xpack.ilm.IndexLifecycleService;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.elasticsearch.xpack.core.ilm.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
+import static org.elasticsearch.xpack.ilm.action.TransportExplainLifecycleAction.getIndexLifecycleExplainResponse;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class TransportExplainLifecycleActionTests extends ESTestCase {
+
+    public static final String PHASE_DEFINITION = "{\n" +
+        "        \"policy\" : \"my-policy\",\n" +
+        "        \"phase_definition\" : {\n" +
+        "          \"min_age\" : \"20m\",\n" +
+        "          \"actions\" : {\n" +
+        "            \"rollover\" : {\n" +
+        "              \"max_age\" : \"5s\"\n" +
+        "            }\n" +
+        "          }\n" +
+        "        },\n" +
+        "        \"version\" : 1,\n" +
+        "        \"modified_date_in_millis\" : 1578521007076\n" +
+        "      }";
+
+    private static final NamedXContentRegistry REGISTRY;
+
+    static {
+        REGISTRY = new NamedXContentRegistry(
+            List.of(new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(RolloverAction.NAME), RolloverAction::parse))
+        );
+    }
+
+    public void testGetIndexLifecycleExplainResponse() throws IOException {
+        {
+            // only errors index
+            IndexLifecycleService indexLifecycleService = mock(IndexLifecycleService.class);
+            when(indexLifecycleService.policyExists("my-policy")).thenReturn(true);
+
+            LifecycleExecutionState.Builder errorStepState = LifecycleExecutionState.builder()
+                .setPhase("hot")
+                .setAction("rollover")
+                .setStep(ErrorStep.NAME)
+                .setPhaseDefinition(PHASE_DEFINITION);
+            String indexInErrorStep = "index_in_error";
+            IndexMetadata meta = IndexMetadata.builder(indexInErrorStep)
+                .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME, "my-policy"))
+                .numberOfShards(randomIntBetween(1, 5))
+                .numberOfReplicas(randomIntBetween(0, 5))
+                .putCustom(ILM_CUSTOM_METADATA_KEY, errorStepState.build().asMap())
+                .build();
+
+            IndexLifecycleExplainResponse onlyErrorsResponse = getIndexLifecycleExplainResponse(meta, true, true, indexLifecycleService,
+                REGISTRY);
+            assertThat(onlyErrorsResponse, notNullValue());
+            assertThat(onlyErrorsResponse.getIndex(), is(indexInErrorStep));
+            assertThat(onlyErrorsResponse.getStep(), is(ErrorStep.NAME));
+        }
+
+        {
+            // only managed index
+            IndexLifecycleService indexLifecycleService = mock(IndexLifecycleService.class);
+            when(indexLifecycleService.policyExists("my-policy")).thenReturn(true);
+
+            LifecycleExecutionState.Builder checkRolloverReadyStepState = LifecycleExecutionState.builder()
+                .setPhase("hot")
+                .setAction("rollover")
+                .setStep(WaitForRolloverReadyStep.NAME)
+                .setPhaseDefinition(PHASE_DEFINITION);
+
+            String indexInCheckRolloverStep = "index_in_check_rollover";
+            IndexMetadata meta = IndexMetadata.builder(indexInCheckRolloverStep)
+                .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME, "my-policy"))
+                .numberOfShards(randomIntBetween(1, 5))
+                .numberOfReplicas(randomIntBetween(0, 5))
+                .putCustom(ILM_CUSTOM_METADATA_KEY, checkRolloverReadyStepState.build().asMap())
+                .build();
+
+            IndexLifecycleExplainResponse onlyErrorsResponse = getIndexLifecycleExplainResponse(meta, true, true, indexLifecycleService,
+                REGISTRY);
+            assertThat(onlyErrorsResponse, nullValue());
+
+            IndexLifecycleExplainResponse allManagedResponse = getIndexLifecycleExplainResponse(meta, false, true, indexLifecycleService,
+                REGISTRY);
+            assertThat(allManagedResponse, notNullValue());
+            assertThat(allManagedResponse.getIndex(), is(indexInCheckRolloverStep));
+            assertThat(allManagedResponse.getStep(), is(WaitForRolloverReadyStep.NAME));
+        }
+
+        {
+            // index with missing policy
+            IndexLifecycleService indexLifecycleService = mock(IndexLifecycleService.class);
+            when(indexLifecycleService.policyExists("random-policy")).thenReturn(false);
+
+            String indexWithMissingPolicy = "index_with_missing_policy";
+            IndexMetadata meta = IndexMetadata.builder(indexWithMissingPolicy)
+                .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME, "random-policy"))
+                .numberOfShards(randomIntBetween(1, 5))
+                .numberOfReplicas(randomIntBetween(0, 5))
+                .build();
+
+            IndexLifecycleExplainResponse onlyErrorsResponse = getIndexLifecycleExplainResponse(meta, true, true, indexLifecycleService,
+                REGISTRY);
+            assertThat(onlyErrorsResponse, notNullValue());
+            assertThat(onlyErrorsResponse.getPolicyName(), is("random-policy"));
+        }
+
+        {
+            // not managed index
+            IndexLifecycleService indexLifecycleService = mock(IndexLifecycleService.class);
+            when(indexLifecycleService.policyExists(anyString())).thenReturn(true);
+
+            IndexMetadata meta = IndexMetadata.builder("index")
+                .settings(settings(Version.CURRENT))
+                .numberOfShards(randomIntBetween(1, 5))
+                .numberOfReplicas(randomIntBetween(0, 5))
+                .build();
+
+            IndexLifecycleExplainResponse onlyManaged = getIndexLifecycleExplainResponse(meta, false, true, indexLifecycleService,
+                REGISTRY);
+            assertThat(onlyManaged, nullValue());
+        }
+    }
+}