Preventing ILM and SLM runtime state from being stored in a snapshot (#92252)

docs/changelog/92252.yaml

@@ -0,0 +1,5 @@
+pr: 92252
+summary: Preventing ILM and SLM runtime state from being stored in a snapshot
+area: ILM+SLM
+type: bug
+issues: []

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java

@@ -55,6 +55,7 @@ import org.elasticsearch.xpack.core.ilm.FreezeAction;
 import org.elasticsearch.xpack.core.ilm.IndexLifecycleFeatureSetUsage;
 import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata;
 import org.elasticsearch.xpack.core.ilm.LifecycleAction;
+import org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata;
 import org.elasticsearch.xpack.core.ilm.LifecycleType;
 import org.elasticsearch.xpack.core.ilm.MigrateAction;
 import org.elasticsearch.xpack.core.ilm.ReadOnlyAction;
@@ -481,6 +482,12 @@ public class XPackClientPlugin extends Plugin implements ActionPlugin, NetworkPl
                 IndexLifecycleMetadata.TYPE,
                 IndexLifecycleMetadata.IndexLifecycleMetadataDiff::new
             ),
+            new NamedWriteableRegistry.Entry(Metadata.Custom.class, LifecycleOperationMetadata.TYPE, LifecycleOperationMetadata::new),
+            new NamedWriteableRegistry.Entry(
+                NamedDiff.class,
+                LifecycleOperationMetadata.TYPE,
+                LifecycleOperationMetadata.LifecycleOperationMetadataDiff::new
+            ),
             new NamedWriteableRegistry.Entry(Metadata.Custom.class, SnapshotLifecycleMetadata.TYPE, SnapshotLifecycleMetadata::new),
             new NamedWriteableRegistry.Entry(
                 NamedDiff.class,

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleMetadata.java

@@ -86,6 +86,10 @@ public class IndexLifecycleMetadata implements Metadata.Custom {
         return policyMetadatas;
     }
 
+    /**
+     * @deprecated use {@link LifecycleOperationMetadata#getILMOperationMode()} instead. This may be incorrect.
+     */
+    @Deprecated(since = "8.7.0")
     public OperationMode getOperationMode() {
         return operationMode;
     }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecycleOperationMetadata.java

@@ -0,0 +1,202 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.ilm;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.cluster.NamedDiff;
+import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.collect.Iterators;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xcontent.ConstructingObjectParser;
+import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.Iterator;
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * Class that encapsulates the running operation mode of Index Lifecycle
+ * Management and Snapshot Lifecycle Management
+ */
+public class LifecycleOperationMetadata implements Metadata.Custom {
+    public static final String TYPE = "lifecycle_operation";
+    public static final ParseField ILM_OPERATION_MODE_FIELD = new ParseField("ilm_operation_mode");
+    public static final ParseField SLM_OPERATION_MODE_FIELD = new ParseField("slm_operation_mode");
+    public static final LifecycleOperationMetadata EMPTY = new LifecycleOperationMetadata(OperationMode.RUNNING, OperationMode.RUNNING);
+
+    @SuppressWarnings("unchecked")
+    public static final ConstructingObjectParser<LifecycleOperationMetadata, Void> PARSER = new ConstructingObjectParser<>(
+        TYPE,
+        a -> new LifecycleOperationMetadata(OperationMode.valueOf((String) a[0]), OperationMode.valueOf((String) a[1]))
+    );
+    static {
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), ILM_OPERATION_MODE_FIELD);
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), SLM_OPERATION_MODE_FIELD);
+    }
+
+    private final OperationMode ilmOperationMode;
+    private final OperationMode slmOperationMode;
+
+    public LifecycleOperationMetadata(OperationMode ilmOperationMode, OperationMode slmOperationMode) {
+        this.ilmOperationMode = ilmOperationMode;
+        this.slmOperationMode = slmOperationMode;
+    }
+
+    public LifecycleOperationMetadata(StreamInput in) throws IOException {
+        this.ilmOperationMode = in.readEnum(OperationMode.class);
+        this.slmOperationMode = in.readEnum(OperationMode.class);
+    }
+
+    /**
+     * Returns the current ILM mode based on the given cluster state. It first checks the newer
+     * storage mechanism ({@link LifecycleOperationMetadata#getILMOperationMode()}) before falling
+     * back to {@link IndexLifecycleMetadata#getOperationMode()}. If neither exist, the default
+     * value for an empty state is used.
+     */
+    @SuppressWarnings("deprecated")
+    public static OperationMode currentILMMode(final ClusterState state) {
+        IndexLifecycleMetadata oldMetadata = state.metadata().custom(IndexLifecycleMetadata.TYPE);
+        LifecycleOperationMetadata currentMetadata = state.metadata().custom(LifecycleOperationMetadata.TYPE);
+        return Optional.ofNullable(currentMetadata)
+            .map(LifecycleOperationMetadata::getILMOperationMode)
+            .orElse(
+                Optional.ofNullable(oldMetadata)
+                    .map(IndexLifecycleMetadata::getOperationMode)
+                    .orElseGet(LifecycleOperationMetadata.EMPTY::getILMOperationMode)
+            );
+    }
+
+    /**
+     * Returns the current ILM mode based on the given cluster state. It first checks the newer
+     * storage mechanism ({@link LifecycleOperationMetadata#getSLMOperationMode()}) before falling
+     * back to {@link SnapshotLifecycleMetadata#getOperationMode()}. If neither exist, the default
+     * value for an empty state is used.
+     */
+    @SuppressWarnings("deprecated")
+    public static OperationMode currentSLMMode(final ClusterState state) {
+        SnapshotLifecycleMetadata oldMetadata = state.metadata().custom(SnapshotLifecycleMetadata.TYPE);
+        LifecycleOperationMetadata currentMetadata = state.metadata().custom(LifecycleOperationMetadata.TYPE);
+        return Optional.ofNullable(currentMetadata)
+            .map(LifecycleOperationMetadata::getSLMOperationMode)
+            .orElse(
+                Optional.ofNullable(oldMetadata)
+                    .map(SnapshotLifecycleMetadata::getOperationMode)
+                    .orElseGet(LifecycleOperationMetadata.EMPTY::getSLMOperationMode)
+            );
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeEnum(ilmOperationMode);
+        out.writeEnum(slmOperationMode);
+    }
+
+    public OperationMode getILMOperationMode() {
+        return ilmOperationMode;
+    }
+
+    public OperationMode getSLMOperationMode() {
+        return slmOperationMode;
+    }
+
+    @Override
+    public Diff<Metadata.Custom> diff(Metadata.Custom previousState) {
+        return new LifecycleOperationMetadata.LifecycleOperationMetadataDiff((LifecycleOperationMetadata) previousState, this);
+    }
+
+    @Override
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
+        ToXContent ilmModeField = ((builder, params2) -> builder.field(ILM_OPERATION_MODE_FIELD.getPreferredName(), ilmOperationMode));
+        ToXContent slmModeField = ((builder, params2) -> builder.field(SLM_OPERATION_MODE_FIELD.getPreferredName(), slmOperationMode));
+        return Iterators.forArray(new ToXContent[] { ilmModeField, slmModeField });
+    }
+
+    @Override
+    public Version getMinimalSupportedVersion() {
+        return Version.V_8_7_0;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return TYPE;
+    }
+
+    @Override
+    public EnumSet<Metadata.XContentContext> context() {
+        // We do not store the lifecycle operation mode for ILM and SLM into a snapshot. This is so
+        // ILM and SLM can be operated independently from restoring snapshots.
+        return EnumSet.of(Metadata.XContentContext.API, Metadata.XContentContext.GATEWAY);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(ilmOperationMode, slmOperationMode);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (obj.getClass() != getClass()) {
+            return false;
+        }
+        LifecycleOperationMetadata other = (LifecycleOperationMetadata) obj;
+        return Objects.equals(ilmOperationMode, other.ilmOperationMode) && Objects.equals(slmOperationMode, other.slmOperationMode);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this, true, true);
+    }
+
+    public static class LifecycleOperationMetadataDiff implements NamedDiff<Metadata.Custom> {
+
+        final OperationMode ilmOperationMode;
+        final OperationMode slmOperationMode;
+
+        LifecycleOperationMetadataDiff(LifecycleOperationMetadata before, LifecycleOperationMetadata after) {
+            this.ilmOperationMode = after.ilmOperationMode;
+            this.slmOperationMode = after.slmOperationMode;
+        }
+
+        public LifecycleOperationMetadataDiff(StreamInput in) throws IOException {
+            this.ilmOperationMode = in.readEnum(OperationMode.class);
+            this.slmOperationMode = in.readEnum(OperationMode.class);
+        }
+
+        @Override
+        public Metadata.Custom apply(Metadata.Custom part) {
+            return new LifecycleOperationMetadata(this.ilmOperationMode, this.slmOperationMode);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeEnum(ilmOperationMode);
+            out.writeEnum(slmOperationMode);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return TYPE;
+        }
+
+        @Override
+        public Version getMinimalSupportedVersion() {
+            return Version.V_8_7_0;
+        }
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleMetadata.java

@@ -21,6 +21,7 @@ import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata;
 import org.elasticsearch.xpack.core.ilm.OperationMode;
 
 import java.io.IOException;
@@ -99,6 +100,10 @@ public class SnapshotLifecycleMetadata implements Metadata.Custom {
         return Collections.unmodifiableMap(this.snapshotConfigurations);
     }
 
+    /**
+     * @deprecated use {@link LifecycleOperationMetadata#getSLMOperationMode()} instead. This may be incorrect.
+     */
+    @Deprecated(since = "8.7.0")
     public OperationMode getOperationMode() {
         return operationMode;
     }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/GetSLMStatusAction.java

@@ -29,7 +29,7 @@ public class GetSLMStatusAction extends ActionType<GetSLMStatusAction.Response>
 
     public static class Response extends ActionResponse implements ToXContentObject {
 
-        private OperationMode mode;
+        private final OperationMode mode;
 
         public Response(StreamInput in) throws IOException {
             super(in);
@@ -45,6 +45,10 @@ public class GetSLMStatusAction extends ActionType<GetSLMStatusAction.Response>
             out.writeEnum(this.mode);
         }
 
+        public OperationMode getOperationMode() {
+            return this.mode;
+        }
+
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecycleOperationMetadataTests.java

@@ -0,0 +1,67 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.ilm;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
+import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.xcontent.XContentParser;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+
+public class LifecycleOperationMetadataTests extends AbstractChunkedSerializingTestCase<Metadata.Custom> {
+
+    @Override
+    protected LifecycleOperationMetadata createTestInstance() {
+        return new LifecycleOperationMetadata(randomFrom(OperationMode.values()), randomFrom(OperationMode.values()));
+    }
+
+    @Override
+    protected LifecycleOperationMetadata doParseInstance(XContentParser parser) throws IOException {
+        return LifecycleOperationMetadata.PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected Writeable.Reader<Metadata.Custom> instanceReader() {
+        return LifecycleOperationMetadata::new;
+    }
+
+    @Override
+    protected Metadata.Custom mutateInstance(Metadata.Custom instance) {
+        LifecycleOperationMetadata metadata = (LifecycleOperationMetadata) instance;
+        if (randomBoolean()) {
+            return new LifecycleOperationMetadata(
+                randomValueOtherThan(metadata.getILMOperationMode(), () -> randomFrom(OperationMode.values())),
+                metadata.getSLMOperationMode()
+            );
+        } else {
+            return new LifecycleOperationMetadata(
+                metadata.getILMOperationMode(),
+                randomValueOtherThan(metadata.getSLMOperationMode(), () -> randomFrom(OperationMode.values()))
+            );
+        }
+    }
+
+    @Override
+    protected boolean isFragment() {
+        return true;
+    }
+
+    public void testMinimumSupportedVersion() {
+        Version min = createTestInstance().getMinimalSupportedVersion();
+        assertTrue(min.onOrBefore(VersionUtils.randomVersionBetween(random(), Version.V_8_7_0, Version.CURRENT)));
+    }
+
+    public void testcontext() {
+        assertThat(createTestInstance().context(), containsInAnyOrder(Metadata.XContentContext.API, Metadata.XContentContext.GATEWAY));
+    }
+}

x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java

@@ -478,12 +478,11 @@ public class IndexLifecycleInitialisationTests extends ESIntegTestCase {
         final String node1 = getLocalNodeId(server_1);
 
         assertAcked(client().execute(StopILMAction.INSTANCE, new StopILMRequest()).get());
-        assertBusy(
-            () -> assertThat(
-                client().execute(GetStatusAction.INSTANCE, new GetStatusAction.Request()).get().getMode(),
-                equalTo(OperationMode.STOPPED)
-            )
-        );
+        assertBusy(() -> {
+            OperationMode mode = client().execute(GetStatusAction.INSTANCE, new GetStatusAction.Request()).get().getMode();
+            logger.info("--> waiting for STOPPED, currently: {}", mode);
+            assertThat(mode, equalTo(OperationMode.STOPPED));
+        });
 
         logger.info("Creating lifecycle [test_lifecycle]");
         PutLifecycleAction.Request putLifecycleRequest = new PutLifecycleAction.Request(lifecyclePolicy);

x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SnapshotLifecycleInitialisationTests.java

@@ -17,7 +17,6 @@ import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.ilm.OperationMode;
-import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata;
 import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicy;
 import org.elasticsearch.xpack.core.slm.SnapshotRetentionConfiguration;
 import org.elasticsearch.xpack.core.slm.action.PutSnapshotLifecycleAction;
@@ -31,6 +30,7 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.concurrent.TimeUnit;
 
+import static org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata.currentSLMMode;
 import static org.hamcrest.core.Is.is;
 
 public class SnapshotLifecycleInitialisationTests extends ESSingleNodeTestCase {
@@ -82,7 +82,6 @@ public class SnapshotLifecycleInitialisationTests extends ESSingleNodeTestCase {
         ).get(10, TimeUnit.SECONDS);
 
         ClusterState state = getInstanceFromNode(ClusterService.class).state();
-        SnapshotLifecycleMetadata snapMeta = state.metadata().custom(SnapshotLifecycleMetadata.TYPE);
-        assertThat(snapMeta.getOperationMode(), is(OperationMode.RUNNING));
+        assertThat(currentSLMMode(state), is(OperationMode.RUNNING));
     }
 }

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java

@@ -51,6 +51,7 @@ import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_REQ
 import static org.elasticsearch.cluster.metadata.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
 import static org.elasticsearch.cluster.routing.allocation.DataTier.ENFORCE_DEFAULT_TIER_PREFERENCE;
 import static org.elasticsearch.cluster.routing.allocation.DataTier.TIER_PREFERENCE;
+import static org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata.currentILMMode;
 import static org.elasticsearch.xpack.core.ilm.OperationMode.STOPPED;
 import static org.elasticsearch.xpack.core.ilm.PhaseCacheManagement.updateIndicesForPolicy;
 import static org.elasticsearch.xpack.ilm.IndexLifecycleTransition.moveStateToNextActionAndUpdateCachedPhase;
@@ -182,9 +183,9 @@ public final class MetadataMigrateToDataTiersRoutingService {
     ) {
         if (dryRun == false) {
             IndexLifecycleMetadata currentMetadata = currentState.metadata().custom(IndexLifecycleMetadata.TYPE);
-            if (currentMetadata != null && currentMetadata.getOperationMode() != STOPPED) {
+            if (currentMetadata != null && currentILMMode(currentState) != STOPPED) {
                 throw new IllegalStateException(
-                    "stop ILM before migrating to data tiers, current state is [" + currentMetadata.getOperationMode() + "]"
+                    "stop ILM before migrating to data tiers, current state is [" + currentILMMode(currentState) + "]"
                 );
             }
         }
@@ -274,7 +275,7 @@ public final class MetadataMigrateToDataTiersRoutingService {
         }
 
         if (migratedPolicies.size() > 0) {
-            IndexLifecycleMetadata newMetadata = new IndexLifecycleMetadata(newPolicies, currentLifecycleMetadata.getOperationMode());
+            IndexLifecycleMetadata newMetadata = new IndexLifecycleMetadata(newPolicies, currentILMMode(currentState));
             mb.putCustom(IndexLifecycleMetadata.TYPE, newMetadata);
         }
         return migratedPolicies;

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IlmHealthIndicatorService.java

@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.ilm;
 
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.health.Diagnosis;
 import org.elasticsearch.health.HealthIndicatorDetails;
@@ -25,6 +26,7 @@ import java.util.Map;
 
 import static org.elasticsearch.health.HealthStatus.GREEN;
 import static org.elasticsearch.health.HealthStatus.YELLOW;
+import static org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata.currentILMMode;
 
 /**
  * This indicator reports health for index lifecycle management component.
@@ -65,16 +67,18 @@ public class IlmHealthIndicatorService implements HealthIndicatorService {
 
     @Override
     public HealthIndicatorResult calculate(boolean verbose, HealthInfo healthInfo) {
-        var ilmMetadata = clusterService.state().metadata().custom(IndexLifecycleMetadata.TYPE, IndexLifecycleMetadata.EMPTY);
+        final ClusterState currentState = clusterService.state();
+        var ilmMetadata = currentState.metadata().custom(IndexLifecycleMetadata.TYPE, IndexLifecycleMetadata.EMPTY);
+        final OperationMode currentMode = currentILMMode(currentState);
         if (ilmMetadata.getPolicyMetadatas().isEmpty()) {
             return createIndicator(
                 GREEN,
                 "No Index Lifecycle Management policies configured",
-                createDetails(verbose, ilmMetadata),
+                createDetails(verbose, ilmMetadata, currentMode),
                 Collections.emptyList(),
                 Collections.emptyList()
             );
-        } else if (ilmMetadata.getOperationMode() != OperationMode.RUNNING) {
+        } else if (currentMode != OperationMode.RUNNING) {
             List<HealthIndicatorImpact> impacts = Collections.singletonList(
                 new HealthIndicatorImpact(
                     NAME,
@@ -88,7 +92,7 @@ public class IlmHealthIndicatorService implements HealthIndicatorService {
             return createIndicator(
                 YELLOW,
                 "Index Lifecycle Management is not running",
-                createDetails(verbose, ilmMetadata),
+                createDetails(verbose, ilmMetadata, currentMode),
                 impacts,
                 List.of(ILM_NOT_RUNNING)
             );
@@ -96,18 +100,16 @@ public class IlmHealthIndicatorService implements HealthIndicatorService {
             return createIndicator(
                 GREEN,
                 "Index Lifecycle Management is running",
-                createDetails(verbose, ilmMetadata),
+                createDetails(verbose, ilmMetadata, currentMode),
                 Collections.emptyList(),
                 Collections.emptyList()
             );
         }
     }
 
-    private static HealthIndicatorDetails createDetails(boolean verbose, IndexLifecycleMetadata metadata) {
+    private static HealthIndicatorDetails createDetails(boolean verbose, IndexLifecycleMetadata metadata, OperationMode mode) {
         if (verbose) {
-            return new SimpleHealthIndicatorDetails(
-                Map.of("ilm_status", metadata.getOperationMode(), "policies", metadata.getPolicies().size())
-            );
+            return new SimpleHealthIndicatorDetails(Map.of("ilm_status", mode, "policies", metadata.getPolicies().size()));
         } else {
             return HealthIndicatorDetails.EMPTY;
         }

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java

@@ -54,6 +54,7 @@ import org.elasticsearch.xpack.core.ilm.ForceMergeAction;
 import org.elasticsearch.xpack.core.ilm.FreezeAction;
 import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata;
 import org.elasticsearch.xpack.core.ilm.LifecycleAction;
+import org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata;
 import org.elasticsearch.xpack.core.ilm.LifecycleSettings;
 import org.elasticsearch.xpack.core.ilm.LifecycleType;
 import org.elasticsearch.xpack.core.ilm.MigrateAction;
@@ -304,6 +305,11 @@ public class IndexLifecycle extends Plugin implements ActionPlugin, HealthPlugin
                 new ParseField(SnapshotLifecycleMetadata.TYPE),
                 parser -> SnapshotLifecycleMetadata.PARSER.parse(parser, null)
             ),
+            new NamedXContentRegistry.Entry(
+                Metadata.Custom.class,
+                new ParseField(LifecycleOperationMetadata.TYPE),
+                parser -> LifecycleOperationMetadata.PARSER.parse(parser, null)
+            ),
             // Lifecycle Types
             new NamedXContentRegistry.Entry(
                 LifecycleType.class,

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleService.java

@@ -61,6 +61,7 @@ import java.util.stream.Collectors;
 import static org.elasticsearch.core.Strings.format;
 import static org.elasticsearch.xpack.core.ilm.IndexLifecycleOriginationDateParser.parseIndexNameAndExtractDate;
 import static org.elasticsearch.xpack.core.ilm.IndexLifecycleOriginationDateParser.shouldParseIndexName;
+import static org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata.currentILMMode;
 
 /**
  * A service which runs the {@link LifecyclePolicy}s associated with indexes.
@@ -167,7 +168,7 @@ public class IndexLifecycleService
 
         final IndexLifecycleMetadata currentMetadata = clusterState.metadata().custom(IndexLifecycleMetadata.TYPE);
         if (currentMetadata != null) {
-            OperationMode currentMode = currentMetadata.getOperationMode();
+            OperationMode currentMode = currentILMMode(clusterState);
             if (OperationMode.STOPPED.equals(currentMode)) {
                 return;
             }
@@ -238,11 +239,15 @@ public class IndexLifecycleService
             }
 
             if (safeToStop && OperationMode.STOPPING == currentMode) {
-                submitUnbatchedTask("ilm_operation_mode_update[stopped]", OperationModeUpdateTask.ilmMode(OperationMode.STOPPED));
+                stopILM();
             }
         }
     }
 
+    private void stopILM() {
+        submitUnbatchedTask("ilm_operation_mode_update[stopped]", OperationModeUpdateTask.ilmMode(OperationMode.STOPPED));
+    }
+
     @Override
     public void beforeIndexAddedToCluster(Index index, Settings indexSettings) {
         if (shouldParseIndexName(indexSettings)) {
@@ -318,10 +323,7 @@ public class IndexLifecycleService
                 });
             }
 
-            final IndexLifecycleMetadata lifecycleMetadata = event.state().metadata().custom(IndexLifecycleMetadata.TYPE);
-            if (lifecycleMetadata != null) {
-                triggerPolicies(event.state(), true);
-            }
+            triggerPolicies(event.state(), true);
         }
     }
 
@@ -370,12 +372,15 @@ public class IndexLifecycleService
     void triggerPolicies(ClusterState clusterState, boolean fromClusterStateChange) {
         IndexLifecycleMetadata currentMetadata = clusterState.metadata().custom(IndexLifecycleMetadata.TYPE);
 
+        OperationMode currentMode = currentILMMode(clusterState);
         if (currentMetadata == null) {
+            if (currentMode == OperationMode.STOPPING) {
+                // There are no policies and ILM is in stopping mode, so stop ILM and get out of here
+                stopILM();
+            }
             return;
         }
 
-        OperationMode currentMode = currentMetadata.getOperationMode();
-
         if (OperationMode.STOPPED.equals(currentMode)) {
             return;
         }
@@ -454,7 +459,7 @@ public class IndexLifecycleService
         }
 
         if (safeToStop && OperationMode.STOPPING == currentMode) {
-            submitUnbatchedTask("ilm_operation_mode_update[stopped]", OperationModeUpdateTask.ilmMode(OperationMode.STOPPED));
+            stopILM();
         }
     }
 

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/OperationModeUpdateTask.java

@@ -17,12 +17,14 @@ import org.elasticsearch.cluster.ack.AckedRequest;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata;
+import org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata;
 import org.elasticsearch.xpack.core.ilm.OperationMode;
-import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata;
 
 import java.util.Objects;
 
+import static org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata.currentILMMode;
+import static org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata.currentSLMMode;
+
 /**
  * This task updates the operation mode state for ILM.
  *
@@ -91,27 +93,26 @@ public class OperationModeUpdateTask extends ClusterStateUpdateTask {
         if (ilmMode == null) {
             return currentState;
         }
-        IndexLifecycleMetadata currentMetadata = currentState.metadata().custom(IndexLifecycleMetadata.TYPE);
-        if (currentMetadata != null && currentMetadata.getOperationMode().isValidChange(ilmMode) == false) {
+
+        final OperationMode currentMode = currentILMMode(currentState);
+        if (currentMode.equals(ilmMode)) {
+            // No need for a new state
             return currentState;
-        } else if (currentMetadata == null) {
-            currentMetadata = IndexLifecycleMetadata.EMPTY;
         }
 
         final OperationMode newMode;
-        if (currentMetadata.getOperationMode().isValidChange(ilmMode)) {
+        if (currentMode.isValidChange(ilmMode)) {
             newMode = ilmMode;
         } else {
-            newMode = currentMetadata.getOperationMode();
+            // The transition is invalid, return the current state
+            return currentState;
         }
 
-        if (newMode.equals(ilmMode) == false) {
-            logger.info("updating ILM operation mode to {}", newMode);
-        }
+        logger.info("updating ILM operation mode to {}", newMode);
         return ClusterState.builder(currentState)
             .metadata(
                 Metadata.builder(currentState.metadata())
-                    .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata(currentMetadata.getPolicyMetadatas(), newMode))
+                    .putCustom(LifecycleOperationMetadata.TYPE, new LifecycleOperationMetadata(newMode, currentSLMMode(currentState)))
             )
             .build();
     }
@@ -120,30 +121,26 @@ public class OperationModeUpdateTask extends ClusterStateUpdateTask {
         if (slmMode == null) {
             return currentState;
         }
-        SnapshotLifecycleMetadata currentMetadata = currentState.metadata().custom(SnapshotLifecycleMetadata.TYPE);
-        if (currentMetadata != null && currentMetadata.getOperationMode().isValidChange(slmMode) == false) {
+
+        final OperationMode currentMode = currentSLMMode(currentState);
+        if (currentMode.equals(slmMode)) {
+            // No need for a new state
             return currentState;
-        } else if (currentMetadata == null) {
-            currentMetadata = SnapshotLifecycleMetadata.EMPTY;
         }
 
         final OperationMode newMode;
-        if (currentMetadata.getOperationMode().isValidChange(slmMode)) {
+        if (currentMode.isValidChange(slmMode)) {
             newMode = slmMode;
         } else {
-            newMode = currentMetadata.getOperationMode();
+            // The transition is invalid, return the current state
+            return currentState;
         }
 
-        if (newMode.equals(slmMode) == false) {
-            logger.info("updating SLM operation mode to {}", newMode);
-        }
+        logger.info("updating SLM operation mode to {}", newMode);
         return ClusterState.builder(currentState)
             .metadata(
                 Metadata.builder(currentState.metadata())
-                    .putCustom(
-                        SnapshotLifecycleMetadata.TYPE,
-                        new SnapshotLifecycleMetadata(currentMetadata.getSnapshotConfigurations(), newMode, currentMetadata.getStats())
-                    )
+                    .putCustom(LifecycleOperationMetadata.TYPE, new LifecycleOperationMetadata(currentILMMode(currentState), newMode))
             )
             .build();
     }

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportDeleteLifecycleAction.java

@@ -37,6 +37,8 @@ import java.util.Set;
 import java.util.SortedMap;
 import java.util.TreeMap;
 
+import static org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata.currentILMMode;
+
 public class TransportDeleteLifecycleAction extends TransportMasterNodeAction<Request, AcknowledgedResponse> {
 
     @Inject
@@ -103,7 +105,7 @@ public class TransportDeleteLifecycleAction extends TransportMasterNodeAction<Re
             }
             SortedMap<String, LifecyclePolicyMetadata> newPolicies = new TreeMap<>(currentMetadata.getPolicyMetadatas());
             newPolicies.remove(request.getPolicyName());
-            IndexLifecycleMetadata newMetadata = new IndexLifecycleMetadata(newPolicies, currentMetadata.getOperationMode());
+            IndexLifecycleMetadata newMetadata = new IndexLifecycleMetadata(newPolicies, currentILMMode(currentState));
             newState.metadata(Metadata.builder(currentState.getMetadata()).putCustom(IndexLifecycleMetadata.TYPE, newMetadata).build());
             return newState.build();
         }

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportGetStatusAction.java

@@ -19,12 +19,12 @@ import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata;
-import org.elasticsearch.xpack.core.ilm.OperationMode;
 import org.elasticsearch.xpack.core.ilm.action.GetStatusAction;
 import org.elasticsearch.xpack.core.ilm.action.GetStatusAction.Request;
 import org.elasticsearch.xpack.core.ilm.action.GetStatusAction.Response;
 
+import static org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata.currentILMMode;
+
 public class TransportGetStatusAction extends TransportMasterNodeAction<Request, Response> {
 
     @Inject
@@ -50,15 +50,7 @@ public class TransportGetStatusAction extends TransportMasterNodeAction<Request,
 
     @Override
     protected void masterOperation(Task task, Request request, ClusterState state, ActionListener<Response> listener) {
-        IndexLifecycleMetadata metadata = state.metadata().custom(IndexLifecycleMetadata.TYPE);
-        final Response response;
-        if (metadata == null) {
-            // no need to actually install metadata just yet, but safe to say it is not stopped
-            response = new Response(OperationMode.RUNNING);
-        } else {
-            response = new Response(metadata.getOperationMode());
-        }
-        listener.onResponse(response);
+        listener.onResponse(new Response(currentILMMode(state)));
     }
 
     @Override

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportMigrateToDataTiersAction.java

@@ -38,6 +38,7 @@ import org.elasticsearch.xpack.cluster.metadata.MetadataMigrateToDataTiersRoutin
 import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata;
 
 import static org.elasticsearch.xpack.cluster.metadata.MetadataMigrateToDataTiersRoutingService.migrateToDataTiersRouting;
+import static org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata.currentILMMode;
 import static org.elasticsearch.xpack.core.ilm.OperationMode.STOPPED;
 
 public class TransportMigrateToDataTiersAction extends TransportMasterNodeAction<MigrateToDataTiersRequest, MigrateToDataTiersResponse> {
@@ -107,11 +108,9 @@ public class TransportMigrateToDataTiersAction extends TransportMasterNodeAction
         }
 
         IndexLifecycleMetadata currentMetadata = state.metadata().custom(IndexLifecycleMetadata.TYPE);
-        if (currentMetadata != null && currentMetadata.getOperationMode() != STOPPED) {
+        if (currentMetadata != null && currentILMMode(state) != STOPPED) {
             listener.onFailure(
-                new IllegalStateException(
-                    "stop ILM before migrating to data tiers, current state is [" + currentMetadata.getOperationMode() + "]"
-                )
+                new IllegalStateException("stop ILM before migrating to data tiers, current state is [" + currentILMMode(state) + "]")
             );
             return;
         }

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleAction.java

@@ -52,6 +52,7 @@ import java.util.Set;
 import java.util.SortedMap;
 import java.util.TreeMap;
 
+import static org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata.currentILMMode;
 import static org.elasticsearch.xpack.core.ilm.PhaseCacheManagement.updateIndicesForPolicy;
 import static org.elasticsearch.xpack.core.searchablesnapshots.SearchableSnapshotsConstants.SEARCHABLE_SNAPSHOT_FEATURE;
 
@@ -190,7 +191,7 @@ public class TransportPutLifecycleAction extends TransportMasterNodeAction<Reque
                     logger.info("updating index lifecycle policy [{}]", request.getPolicy().getName());
                 }
             }
-            IndexLifecycleMetadata newMetadata = new IndexLifecycleMetadata(newPolicies, currentMetadata.getOperationMode());
+            IndexLifecycleMetadata newMetadata = new IndexLifecycleMetadata(newPolicies, currentILMMode(currentState));
             stateBuilder.metadata(Metadata.builder(currentState.getMetadata()).putCustom(IndexLifecycleMetadata.TYPE, newMetadata).build());
             ClusterState nonRefreshedState = stateBuilder.build();
             if (oldPolicy == null) {

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SlmHealthIndicatorService.java

@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.slm;
 
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.health.Diagnosis;
@@ -32,6 +33,7 @@ import java.util.stream.Collectors;
 
 import static org.elasticsearch.health.HealthStatus.GREEN;
 import static org.elasticsearch.health.HealthStatus.YELLOW;
+import static org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata.currentSLMMode;
 import static org.elasticsearch.xpack.core.ilm.LifecycleSettings.SLM_HEALTH_FAILED_SNAPSHOT_WARN_THRESHOLD_SETTING;
 
 /**
@@ -98,16 +100,18 @@ public class SlmHealthIndicatorService implements HealthIndicatorService {
 
     @Override
     public HealthIndicatorResult calculate(boolean verbose, HealthInfo healthInfo) {
-        var slmMetadata = clusterService.state().metadata().custom(SnapshotLifecycleMetadata.TYPE, SnapshotLifecycleMetadata.EMPTY);
+        final ClusterState currentState = clusterService.state();
+        var slmMetadata = currentState.metadata().custom(SnapshotLifecycleMetadata.TYPE, SnapshotLifecycleMetadata.EMPTY);
+        final OperationMode currentMode = currentSLMMode(currentState);
         if (slmMetadata.getSnapshotConfigurations().isEmpty()) {
             return createIndicator(
                 GREEN,
                 "No Snapshot Lifecycle Management policies configured",
-                createDetails(verbose, Collections.emptyList(), slmMetadata),
+                createDetails(verbose, Collections.emptyList(), slmMetadata, currentMode),
                 Collections.emptyList(),
                 Collections.emptyList()
             );
-        } else if (slmMetadata.getOperationMode() != OperationMode.RUNNING) {
+        } else if (currentMode != OperationMode.RUNNING) {
             List<HealthIndicatorImpact> impacts = Collections.singletonList(
                 new HealthIndicatorImpact(
                     NAME,
@@ -120,7 +124,7 @@ public class SlmHealthIndicatorService implements HealthIndicatorService {
             return createIndicator(
                 YELLOW,
                 "Snapshot Lifecycle Management is not running",
-                createDetails(verbose, Collections.emptyList(), slmMetadata),
+                createDetails(verbose, Collections.emptyList(), slmMetadata, currentMode),
                 impacts,
                 List.of(SLM_NOT_RUNNING)
             );
@@ -169,7 +173,7 @@ public class SlmHealthIndicatorService implements HealthIndicatorService {
                 return createIndicator(
                     YELLOW,
                     "Encountered [" + unhealthyPolicies.size() + "] unhealthy snapshot lifecycle management policies.",
-                    createDetails(verbose, unhealthyPolicies, slmMetadata),
+                    createDetails(verbose, unhealthyPolicies, slmMetadata, currentMode),
                     impacts,
                     List.of(
                         new Diagnosis(
@@ -188,7 +192,7 @@ public class SlmHealthIndicatorService implements HealthIndicatorService {
             return createIndicator(
                 GREEN,
                 "Snapshot Lifecycle Management is running",
-                createDetails(verbose, Collections.emptyList(), slmMetadata),
+                createDetails(verbose, Collections.emptyList(), slmMetadata, currentMode),
                 Collections.emptyList(),
                 Collections.emptyList()
             );
@@ -215,11 +219,12 @@ public class SlmHealthIndicatorService implements HealthIndicatorService {
     private static HealthIndicatorDetails createDetails(
         boolean verbose,
         Collection<SnapshotLifecyclePolicyMetadata> unhealthyPolicies,
-        SnapshotLifecycleMetadata metadata
+        SnapshotLifecycleMetadata metadata,
+        OperationMode mode
     ) {
         if (verbose) {
             Map<String, Object> details = new LinkedHashMap<>();
-            details.put("slm_status", metadata.getOperationMode());
+            details.put("slm_status", mode);
             details.put("policies", metadata.getSnapshotConfigurations().size());
             if (unhealthyPolicies.size() > 0) {
                 details.put(

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleService.java

@@ -31,13 +31,14 @@ import org.elasticsearch.xpack.ilm.OperationModeUpdateTask;
 import java.io.Closeable;
 import java.time.Clock;
 import java.util.Map;
-import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Supplier;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata.currentSLMMode;
+
 /**
  * {@code SnapshotLifecycleService} manages snapshot policy scheduling and triggering of the
  * {@link SnapshotLifecycleTask}. It reacts to new policies in the cluster state by scheduling a
@@ -121,20 +122,16 @@ public class SnapshotLifecycleService implements Closeable, ClusterStateListener
      * Returns true if SLM is in the stopping or stopped state
      */
     static boolean slmStoppedOrStopping(ClusterState state) {
-        return Optional.ofNullable((SnapshotLifecycleMetadata) state.metadata().custom(SnapshotLifecycleMetadata.TYPE))
-            .map(SnapshotLifecycleMetadata::getOperationMode)
-            .map(mode -> OperationMode.STOPPING == mode || OperationMode.STOPPED == mode)
-            .orElse(false);
+        OperationMode mode = currentSLMMode(state);
+        return OperationMode.STOPPING == mode || OperationMode.STOPPED == mode;
     }
 
     /**
      * Returns true if SLM is in the stopping state
      */
     static boolean slmStopping(ClusterState state) {
-        return Optional.ofNullable((SnapshotLifecycleMetadata) state.metadata().custom(SnapshotLifecycleMetadata.TYPE))
-            .map(SnapshotLifecycleMetadata::getOperationMode)
-            .map(mode -> OperationMode.STOPPING == mode)
-            .orElse(false);
+        OperationMode mode = currentSLMMode(state);
+        return OperationMode.STOPPING == mode;
     }
 
     /**

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTask.java

@@ -41,6 +41,7 @@ import java.util.Map;
 import java.util.Optional;
 
 import static org.elasticsearch.core.Strings.format;
+import static org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata.currentSLMMode;
 
 public class SnapshotLifecycleTask implements SchedulerEngine.Listener {
 
@@ -284,7 +285,11 @@ public class SnapshotLifecycleTask implements SchedulerEngine.Listener {
             }
 
             snapLifecycles.put(policyName, newPolicyMetadata.build());
-            SnapshotLifecycleMetadata lifecycleMetadata = new SnapshotLifecycleMetadata(snapLifecycles, snapMeta.getOperationMode(), stats);
+            SnapshotLifecycleMetadata lifecycleMetadata = new SnapshotLifecycleMetadata(
+                snapLifecycles,
+                currentSLMMode(currentState),
+                stats
+            );
             Metadata currentMeta = currentState.metadata();
             return ClusterState.builder(currentState)
                 .metadata(Metadata.builder(currentMeta).putCustom(SnapshotLifecycleMetadata.TYPE, lifecycleMetadata))

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/UpdateSnapshotLifecycleStatsTask.java

@@ -16,6 +16,7 @@ import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata;
 import org.elasticsearch.xpack.core.slm.SnapshotLifecycleStats;
 
 import static org.elasticsearch.core.Strings.format;
+import static org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata.currentSLMMode;
 
 /**
  * {@link UpdateSnapshotLifecycleStatsTask} is a cluster state update task that retrieves the
@@ -45,7 +46,7 @@ public class UpdateSnapshotLifecycleStatsTask extends ClusterStateUpdateTask {
         SnapshotLifecycleStats newMetrics = currentSlmMeta.getStats().merge(runStats);
         SnapshotLifecycleMetadata newSlmMeta = new SnapshotLifecycleMetadata(
             currentSlmMeta.getSnapshotConfigurations(),
-            currentSlmMeta.getOperationMode(),
+            currentSLMMode(currentState),
             newMetrics
         );
 

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/TransportDeleteSnapshotLifecycleAction.java

@@ -26,6 +26,7 @@ import org.elasticsearch.reservedstate.ReservedClusterStateHandler;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata;
 import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata;
 import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicyMetadata;
 import org.elasticsearch.xpack.core.slm.action.DeleteSnapshotLifecycleAction;
@@ -101,6 +102,7 @@ public class TransportDeleteSnapshotLifecycleAction extends TransportMasterNodeA
             if (snapMeta == null) {
                 throw new ResourceNotFoundException("snapshot lifecycle policy not found: {}", request.getLifecycleId());
             }
+            var currentMode = LifecycleOperationMetadata.currentSLMMode(currentState);
             // Check that the policy exists in the first place
             snapMeta.getSnapshotConfigurations()
                 .entrySet()
@@ -123,7 +125,7 @@ public class TransportDeleteSnapshotLifecycleAction extends TransportMasterNodeA
                             SnapshotLifecycleMetadata.TYPE,
                             new SnapshotLifecycleMetadata(
                                 newConfigs,
-                                snapMeta.getOperationMode(),
+                                currentMode,
                                 snapMeta.getStats().removePolicy(request.getLifecycleId())
                             )
                         )

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/TransportGetSLMStatusAction.java

@@ -19,10 +19,10 @@ import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.xpack.core.ilm.OperationMode;
-import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata;
 import org.elasticsearch.xpack.core.slm.action.GetSLMStatusAction;
 
+import static org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata.currentSLMMode;
+
 public class TransportGetSLMStatusAction extends TransportMasterNodeAction<GetSLMStatusAction.Request, GetSLMStatusAction.Response> {
 
     @Inject
@@ -53,15 +53,7 @@ public class TransportGetSLMStatusAction extends TransportMasterNodeAction<GetSL
         ClusterState state,
         ActionListener<GetSLMStatusAction.Response> listener
     ) {
-        SnapshotLifecycleMetadata metadata = state.metadata().custom(SnapshotLifecycleMetadata.TYPE);
-        final GetSLMStatusAction.Response response;
-        if (metadata == null) {
-            // no need to actually install metadata just yet, but safe to say it is not stopped
-            response = new GetSLMStatusAction.Response(OperationMode.RUNNING);
-        } else {
-            response = new GetSLMStatusAction.Response(metadata.getOperationMode());
-        }
-        listener.onResponse(response);
+        listener.onResponse(new GetSLMStatusAction.Response(currentSLMMode(state)));
     }
 
     @Override

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/TransportPutSnapshotLifecycleAction.java

@@ -28,8 +28,8 @@ import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ClientHelper;
+import org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata;
 import org.elasticsearch.xpack.core.ilm.LifecyclePolicy;
-import org.elasticsearch.xpack.core.ilm.OperationMode;
 import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata;
 import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicyMetadata;
 import org.elasticsearch.xpack.core.slm.SnapshotLifecycleStats;
@@ -129,6 +129,7 @@ public class TransportPutSnapshotLifecycleAction extends TransportMasterNodeActi
         @Override
         public ClusterState execute(ClusterState currentState) {
             SnapshotLifecycleMetadata snapMeta = currentState.metadata().custom(SnapshotLifecycleMetadata.TYPE);
+            var currentMode = LifecycleOperationMetadata.currentSLMMode(currentState);
 
             String id = request.getLifecycleId();
             final SnapshotLifecycleMetadata lifecycleMetadata;
@@ -140,7 +141,7 @@ public class TransportPutSnapshotLifecycleAction extends TransportMasterNodeActi
                     .build();
                 lifecycleMetadata = new SnapshotLifecycleMetadata(
                     Collections.singletonMap(id, meta),
-                    OperationMode.RUNNING,
+                    currentMode,
                     new SnapshotLifecycleStats()
                 );
                 logger.info("adding new snapshot lifecycle [{}]", id);
@@ -154,7 +155,7 @@ public class TransportPutSnapshotLifecycleAction extends TransportMasterNodeActi
                     .setModifiedDate(Instant.now().toEpochMilli())
                     .build();
                 snapLifecycles.put(id, newLifecycle);
-                lifecycleMetadata = new SnapshotLifecycleMetadata(snapLifecycles, snapMeta.getOperationMode(), snapMeta.getStats());
+                lifecycleMetadata = new SnapshotLifecycleMetadata(snapLifecycles, currentMode, snapMeta.getStats());
                 if (oldLifecycle == null) {
                     logger.info("adding new snapshot lifecycle [{}]", id);
                 } else {

x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/LifecycleOperationSnapshotTests.java

@@ -0,0 +1,137 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.ilm;
+
+import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsAction;
+import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
+import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse;
+import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotAction;
+import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.snapshots.SnapshotState;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
+import org.elasticsearch.xpack.core.ilm.LifecyclePolicy;
+import org.elasticsearch.xpack.core.ilm.OperationMode;
+import org.elasticsearch.xpack.core.ilm.Phase;
+import org.elasticsearch.xpack.core.ilm.ReadOnlyAction;
+import org.elasticsearch.xpack.core.ilm.StopILMRequest;
+import org.elasticsearch.xpack.core.ilm.action.GetStatusAction;
+import org.elasticsearch.xpack.core.ilm.action.PutLifecycleAction;
+import org.elasticsearch.xpack.core.ilm.action.StopILMAction;
+import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicy;
+import org.elasticsearch.xpack.core.slm.action.ExecuteSnapshotLifecycleAction;
+import org.elasticsearch.xpack.core.slm.action.GetSLMStatusAction;
+import org.elasticsearch.xpack.core.slm.action.PutSnapshotLifecycleAction;
+import org.elasticsearch.xpack.core.slm.action.StopSLMAction;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicyMetadataTests.randomRetention;
+import static org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicyMetadataTests.randomSchedule;
+import static org.hamcrest.Matchers.equalTo;
+
+public class LifecycleOperationSnapshotTests extends ESSingleNodeTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return List.of(LocalStateCompositeXPackPlugin.class, IndexLifecycle.class);
+    }
+
+    @Override
+    protected Settings nodeSettings() {
+        return Settings.builder().put(super.nodeSettings()).put("slm.history_index_enabled", false).build();
+    }
+
+    public void testModeSnapshotRestore() throws Exception {
+        client().admin()
+            .cluster()
+            .preparePutRepository("repo")
+            .setType("fs")
+            .setSettings(Settings.builder().put("location", "repo").build())
+            .get();
+
+        client().execute(
+            PutSnapshotLifecycleAction.INSTANCE,
+            new PutSnapshotLifecycleAction.Request(
+                "slm-policy",
+                new SnapshotLifecyclePolicy(
+                    "slm-policy",
+                    randomAlphaOfLength(4).toLowerCase(Locale.ROOT),
+                    randomSchedule(),
+                    "repo",
+                    null,
+                    randomRetention()
+                )
+            )
+        ).get();
+
+        client().execute(
+            PutLifecycleAction.INSTANCE,
+            new PutLifecycleAction.Request(
+                new LifecyclePolicy(
+                    "ilm-policy",
+                    Map.of("warm", new Phase("warm", TimeValue.timeValueHours(1), Map.of("readonly", new ReadOnlyAction())))
+                )
+            )
+        );
+
+        assertThat(ilmMode(), equalTo(OperationMode.RUNNING));
+        assertThat(slmMode(), equalTo(OperationMode.RUNNING));
+
+        // Take snapshot
+        ExecuteSnapshotLifecycleAction.Response resp = client().execute(
+            ExecuteSnapshotLifecycleAction.INSTANCE,
+            new ExecuteSnapshotLifecycleAction.Request("slm-policy")
+        ).get();
+        final String snapshotName = resp.getSnapshotName();
+        // Wait for the snapshot to be successful
+        assertBusy(() -> {
+            logger.info("--> checking for snapshot success");
+            try {
+                GetSnapshotsResponse getResp = client().execute(
+                    GetSnapshotsAction.INSTANCE,
+                    new GetSnapshotsRequest(new String[] { "repo" }, new String[] { snapshotName })
+                ).get();
+                assertThat(getResp.getSnapshots().size(), equalTo(1));
+                assertThat(getResp.getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
+            } catch (Exception e) {
+                fail("snapshot does not yet exist");
+            }
+        });
+
+        assertAcked(client().execute(StopILMAction.INSTANCE, new StopILMRequest()).get());
+        assertAcked(client().execute(StopSLMAction.INSTANCE, new StopSLMAction.Request()).get());
+        assertBusy(() -> assertThat(ilmMode(), equalTo(OperationMode.STOPPED)));
+        assertBusy(() -> assertThat(slmMode(), equalTo(OperationMode.STOPPED)));
+
+        // Restore snapshot
+        client().execute(
+            RestoreSnapshotAction.INSTANCE,
+            new RestoreSnapshotRequest("repo", snapshotName).includeGlobalState(true).indices(Strings.EMPTY_ARRAY).waitForCompletion(true)
+        ).get();
+
+        assertBusy(() -> assertThat(ilmMode(), equalTo(OperationMode.STOPPED)));
+        assertBusy(() -> assertThat(slmMode(), equalTo(OperationMode.STOPPED)));
+    }
+
+    private OperationMode ilmMode() throws Exception {
+        return client().execute(GetStatusAction.INSTANCE, new GetStatusAction.Request()).get().getMode();
+    }
+
+    private OperationMode slmMode() throws Exception {
+        return client().execute(GetSLMStatusAction.INSTANCE, new GetSLMStatusAction.Request()).get().getOperationMode();
+    }
+}

x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/OperationModeUpdateTaskTests.java

@@ -12,50 +12,89 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata;
+import org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata;
 import org.elasticsearch.xpack.core.ilm.OperationMode;
 import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata;
 import org.elasticsearch.xpack.core.slm.SnapshotLifecycleStats;
 
 import java.util.Collections;
 import java.util.Map;
+import java.util.Optional;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.not;
 
 public class OperationModeUpdateTaskTests extends ESTestCase {
 
-    public void testExecute() {
-        assertMove(OperationMode.RUNNING, OperationMode.STOPPING);
-        assertMove(OperationMode.STOPPING, randomFrom(OperationMode.RUNNING, OperationMode.STOPPED));
-        assertMove(OperationMode.STOPPED, OperationMode.RUNNING);
+    public void testILMExecute() {
+        assertILMMove(OperationMode.RUNNING, OperationMode.STOPPING);
+        assertILMMove(OperationMode.STOPPING, OperationMode.RUNNING);
+        assertILMMove(OperationMode.STOPPING, OperationMode.STOPPED);
+        assertILMMove(OperationMode.STOPPED, OperationMode.RUNNING);
 
         OperationMode mode = randomFrom(OperationMode.values());
-        assertNoMove(mode, mode);
-        assertNoMove(OperationMode.STOPPED, OperationMode.STOPPING);
-        assertNoMove(OperationMode.RUNNING, OperationMode.STOPPED);
+        assertNoILMMove(mode, mode);
+        assertNoILMMove(OperationMode.STOPPED, OperationMode.STOPPING);
+        assertNoILMMove(OperationMode.RUNNING, OperationMode.STOPPED);
+    }
+
+    public void testSLMExecute() {
+        assertSLMMove(OperationMode.RUNNING, OperationMode.STOPPING);
+        assertSLMMove(OperationMode.STOPPING, OperationMode.RUNNING);
+        assertSLMMove(OperationMode.STOPPING, OperationMode.STOPPED);
+        assertSLMMove(OperationMode.STOPPED, OperationMode.RUNNING);
+
+        OperationMode mode = randomFrom(OperationMode.values());
+        assertNoSLMMove(mode, mode);
+        assertNoSLMMove(OperationMode.STOPPED, OperationMode.STOPPING);
+        assertNoSLMMove(OperationMode.RUNNING, OperationMode.STOPPED);
     }
 
     public void testExecuteWithEmptyMetadata() {
         OperationMode requestedMode = OperationMode.STOPPING;
-        OperationMode newMode = executeUpdate(false, IndexLifecycleMetadata.EMPTY.getOperationMode(), requestedMode, false);
+        OperationMode newMode = executeILMUpdate(false, LifecycleOperationMetadata.EMPTY.getILMOperationMode(), requestedMode, false);
+        assertThat(newMode, equalTo(requestedMode));
+
+        newMode = executeSLMUpdate(false, LifecycleOperationMetadata.EMPTY.getSLMOperationMode(), requestedMode, false);
         assertThat(newMode, equalTo(requestedMode));
 
-        requestedMode = randomFrom(OperationMode.RUNNING, OperationMode.STOPPED);
-        newMode = executeUpdate(false, IndexLifecycleMetadata.EMPTY.getOperationMode(), requestedMode, false);
+        requestedMode = OperationMode.RUNNING;
+        newMode = executeILMUpdate(false, LifecycleOperationMetadata.EMPTY.getILMOperationMode(), requestedMode, true);
+        assertThat(newMode, equalTo(OperationMode.RUNNING));
+        requestedMode = OperationMode.STOPPED;
+        newMode = executeILMUpdate(false, LifecycleOperationMetadata.EMPTY.getILMOperationMode(), requestedMode, true);
+        assertThat(newMode, equalTo(OperationMode.RUNNING));
+
+        requestedMode = OperationMode.RUNNING;
+        newMode = executeSLMUpdate(false, LifecycleOperationMetadata.EMPTY.getSLMOperationMode(), requestedMode, true);
+        assertThat(newMode, equalTo(OperationMode.RUNNING));
+        requestedMode = OperationMode.STOPPED;
+        newMode = executeSLMUpdate(false, LifecycleOperationMetadata.EMPTY.getSLMOperationMode(), requestedMode, true);
         assertThat(newMode, equalTo(OperationMode.RUNNING));
     }
 
-    private void assertMove(OperationMode currentMode, OperationMode requestedMode) {
-        OperationMode newMode = executeUpdate(true, currentMode, requestedMode, false);
+    private void assertILMMove(OperationMode currentMode, OperationMode requestedMode) {
+        OperationMode newMode = executeILMUpdate(true, currentMode, requestedMode, false);
         assertThat(newMode, equalTo(requestedMode));
     }
 
-    private void assertNoMove(OperationMode currentMode, OperationMode requestedMode) {
-        OperationMode newMode = executeUpdate(true, currentMode, requestedMode, true);
+    private void assertSLMMove(OperationMode currentMode, OperationMode requestedMode) {
+        OperationMode newMode = executeSLMUpdate(true, currentMode, requestedMode, false);
+        assertThat(newMode, equalTo(requestedMode));
+    }
+
+    private void assertNoILMMove(OperationMode currentMode, OperationMode requestedMode) {
+        OperationMode newMode = executeILMUpdate(true, currentMode, requestedMode, true);
+        assertThat(newMode, equalTo(currentMode));
+    }
+
+    private void assertNoSLMMove(OperationMode currentMode, OperationMode requestedMode) {
+        OperationMode newMode = executeSLMUpdate(true, currentMode, requestedMode, true);
         assertThat(newMode, equalTo(currentMode));
     }
 
-    private OperationMode executeUpdate(
+    @SuppressWarnings("deprecated")
+    private OperationMode executeILMUpdate(
         boolean metadataInstalled,
         OperationMode currentMode,
         OperationMode requestMode,
@@ -76,13 +115,49 @@ public class OperationModeUpdateTaskTests extends ESTestCase {
         ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build();
         OperationModeUpdateTask task = OperationModeUpdateTask.ilmMode(requestMode);
         ClusterState newState = task.execute(state);
+        if (assertSameClusterState) {
+            assertSame("expected the same state instance but they were different", state, newState);
+        } else {
+            assertThat("expected a different state instance but they were the same", state, not(equalTo(newState)));
+        }
+        LifecycleOperationMetadata newMetadata = newState.metadata().custom(LifecycleOperationMetadata.TYPE);
+        IndexLifecycleMetadata oldMetadata = newState.metadata().custom(IndexLifecycleMetadata.TYPE, IndexLifecycleMetadata.EMPTY);
+        return Optional.ofNullable(newMetadata)
+            .map(LifecycleOperationMetadata::getILMOperationMode)
+            .orElseGet(oldMetadata::getOperationMode);
+    }
+
+    @SuppressWarnings("deprecated")
+    private OperationMode executeSLMUpdate(
+        boolean metadataInstalled,
+        OperationMode currentMode,
+        OperationMode requestMode,
+        boolean assertSameClusterState
+    ) {
+        IndexLifecycleMetadata indexLifecycleMetadata = new IndexLifecycleMetadata(Collections.emptyMap(), currentMode);
+        SnapshotLifecycleMetadata snapshotLifecycleMetadata = new SnapshotLifecycleMetadata(
+            Collections.emptyMap(),
+            currentMode,
+            new SnapshotLifecycleStats()
+        );
+        Metadata.Builder metadata = Metadata.builder().persistentSettings(settings(Version.CURRENT).build());
+        if (metadataInstalled) {
+            metadata.customs(
+                Map.of(IndexLifecycleMetadata.TYPE, indexLifecycleMetadata, SnapshotLifecycleMetadata.TYPE, snapshotLifecycleMetadata)
+            );
+        }
+        ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build();
+        OperationModeUpdateTask task = OperationModeUpdateTask.slmMode(requestMode);
+        ClusterState newState = task.execute(state);
         if (assertSameClusterState) {
             assertSame(state, newState);
         } else {
             assertThat(state, not(equalTo(newState)));
         }
-        IndexLifecycleMetadata newMetadata = newState.metadata().custom(IndexLifecycleMetadata.TYPE);
-        assertThat(newMetadata.getPolicyMetadatas(), equalTo(indexLifecycleMetadata.getPolicyMetadatas()));
-        return newMetadata.getOperationMode();
+        LifecycleOperationMetadata newMetadata = newState.metadata().custom(LifecycleOperationMetadata.TYPE);
+        SnapshotLifecycleMetadata oldMetadata = newState.metadata().custom(SnapshotLifecycleMetadata.TYPE, SnapshotLifecycleMetadata.EMPTY);
+        return Optional.ofNullable(newMetadata)
+            .map(LifecycleOperationMetadata::getSLMOperationMode)
+            .orElseGet(oldMetadata::getOperationMode);
     }
 }