Add 3 new settings to configure ILMHealthIndicator (#96947)

Add the following dynamic and cluster-wide settings, to configure the
ILM Health indicator:

* `health.ilm.max_time_on_action`
* `health.ilm.max_time_on_step`
* `health.ilm.max_retries_per_step`

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IlmHealthIndicatorService.java

@@ -9,6 +9,8 @@ package org.elasticsearch.xpack.ilm;
 
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.health.Diagnosis;
 import org.elasticsearch.health.HealthIndicatorDetails;
@@ -44,6 +46,8 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeSet;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
 import java.util.function.LongSupplier;
 import java.util.stream.Collectors;
 
@@ -79,6 +83,30 @@ public class IlmHealthIndicatorService implements HealthIndicatorService {
         null
     );
 
+    static final Setting<TimeValue> MAX_TIME_ON_ACTION_SETTING = Setting.timeSetting(
+        "health.ilm.max_time_on_action",
+        new TimeValue(1, TimeUnit.DAYS),
+        new TimeValue(1, TimeUnit.DAYS),
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
+    );
+
+    static final Setting<TimeValue> MAX_TIME_ON_STEP_SETTING = Setting.timeSetting(
+        "health.ilm.max_time_on_step",
+        new TimeValue(1, TimeUnit.DAYS),
+        new TimeValue(1, TimeUnit.DAYS),
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
+    );
+
+    static final Setting<Long> MAX_RETRIES_PER_STEP_SETTING = Setting.longSetting(
+        "health.ilm.max_retries_per_step",
+        100,
+        2,
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
+    );
+
     public static final String AUTOMATION_DISABLED_IMPACT_ID = "automation_disabled";
     public static final String STAGNATING_INDEX_IMPACT_ID = "stagnating_index";
     public static final List<HealthIndicatorImpact> AUTOMATION_DISABLED_IMPACT = List.of(
@@ -103,62 +131,63 @@ public class IlmHealthIndicatorService implements HealthIndicatorService {
         )
     );
 
-    private static final TimeValue ONE_DAY = TimeValue.timeValueDays(1);
-    private static final long MAX_RETRIES = 100;
-
-    static final Map<String, RuleConfig> RULES_BY_ACTION_CONFIG = Map.of(
+    static final Map<String, RuleCreator> RULES_BY_ACTION_CONFIG = Map.of(
         RolloverAction.NAME,
-        actionRule(RolloverAction.NAME).stepRules(
-            stepRuleFullChecks(WaitForActiveShardsStep.NAME, ONE_DAY, MAX_RETRIES),
-            stepRuleOnlyCheckRetries(WaitForRolloverReadyStep.NAME, MAX_RETRIES),
-            stepRuleFullChecks(RolloverStep.NAME, ONE_DAY, MAX_RETRIES)
+        (maxTimeOnAction, maxTimeOnStep, maxRetries) -> actionRule(RolloverAction.NAME).stepRules(
+            stepRuleFullChecks(WaitForActiveShardsStep.NAME, maxTimeOnStep, maxRetries),
+            stepRuleOnlyCheckRetries(WaitForRolloverReadyStep.NAME, maxRetries),
+            stepRuleFullChecks(RolloverStep.NAME, maxTimeOnStep, maxRetries)
         ),
         //
         MigrateAction.NAME,
-        actionRule(MigrateAction.NAME).maxTimeOnAction(ONE_DAY).noStepRules(),
+        (maxTimeOnAction, maxTimeOnStep, maxRetries) -> actionRule(MigrateAction.NAME).maxTimeOnAction(maxTimeOnAction).noStepRules(),
         //
         SearchableSnapshotAction.NAME,
-        actionRule(SearchableSnapshotAction.NAME).maxTimeOnAction(ONE_DAY)
+        (maxTimeOnAction, maxTimeOnStep, maxRetries) -> actionRule(SearchableSnapshotAction.NAME).maxTimeOnAction(maxTimeOnAction)
             .stepRules(
-                stepRuleFullChecks(WaitForDataTierStep.NAME, ONE_DAY, MAX_RETRIES),
-                stepRuleFullChecks(WaitForIndexColorStep.NAME, ONE_DAY, MAX_RETRIES),
-                stepRuleOnlyCheckRetries(WaitForNoFollowersStep.NAME, MAX_RETRIES)
+                stepRuleFullChecks(WaitForDataTierStep.NAME, maxTimeOnStep, maxRetries),
+                stepRuleFullChecks(WaitForIndexColorStep.NAME, maxTimeOnStep, maxRetries),
+                stepRuleOnlyCheckRetries(WaitForNoFollowersStep.NAME, maxRetries)
             ),
         //
         DeleteAction.NAME,
-        actionRule(DeleteAction.NAME).stepRules(stepRuleFullChecks(DeleteStep.NAME, ONE_DAY, MAX_RETRIES)),
+        (maxTimeOnAction, maxTimeOnStep, maxRetries) -> actionRule(DeleteAction.NAME).stepRules(
+            stepRuleFullChecks(DeleteStep.NAME, maxTimeOnStep, maxRetries)
+        ),
         //
         ShrinkAction.NAME,
-        actionRule(ShrinkAction.NAME).maxTimeOnAction(ONE_DAY)
-            .stepRules(stepRuleOnlyCheckRetries(WaitForNoFollowersStep.NAME, MAX_RETRIES)),
+        (maxTimeOnAction, maxTimeOnStep, maxRetries) -> actionRule(ShrinkAction.NAME).maxTimeOnAction(
+            maxTimeOnAction
+
+        ).stepRules(stepRuleOnlyCheckRetries(WaitForNoFollowersStep.NAME, maxRetries)),
         //
         AllocateAction.NAME,
-        actionRule(AllocateAction.NAME).maxTimeOnAction(ONE_DAY).noStepRules(),
+        (maxTimeOnAction, maxTimeOnStep, maxRetries) -> actionRule(AllocateAction.NAME).maxTimeOnAction(maxTimeOnAction).noStepRules(),
         //
         ForceMergeAction.NAME,
-        actionRule(ForceMergeAction.NAME).maxTimeOnAction(ONE_DAY)
+        (maxTimeOnAction, maxTimeOnStep, maxRetries) -> actionRule(ForceMergeAction.NAME).maxTimeOnAction(maxTimeOnAction)
             .stepRules(
-                stepRuleFullChecks(WaitForIndexColorStep.NAME, ONE_DAY, MAX_RETRIES),
-                stepRuleFullChecks(ForceMergeStep.NAME, ONE_DAY, MAX_RETRIES),
-                stepRuleFullChecks(SegmentCountStep.NAME, ONE_DAY, MAX_RETRIES)
+                stepRuleFullChecks(WaitForIndexColorStep.NAME, maxTimeOnStep, maxRetries),
+                stepRuleFullChecks(ForceMergeStep.NAME, maxTimeOnStep, maxRetries),
+                stepRuleFullChecks(SegmentCountStep.NAME, maxTimeOnStep, maxRetries)
             )
         //
         // The next rule has to be commented because of this issue https://github.com/elastic/elasticsearch/issues/96705
         // DownsampleAction.NAME,
-        // actionRule(DownsampleAction.NAME).maxTimeOnAction(ONE_DAY).stepRules(stepRule(WaitForNoFollowersStep.NAME, ONE_DAY))
+        // (maxTimeOnAction, maxTimeOnStep, maxRetries) ->
+        // actionRule(DownsampleAction.NAME).maxTimeOnAction(maxTimeOnAction).stepRules(stepRule(WaitForNoFollowersStep.NAME,
+        // maxTimeOnAction))
     );
 
-    public static final Collection<RuleConfig> ILM_RULE_EVALUATOR = RULES_BY_ACTION_CONFIG.values();
-
-    static final Map<String, Diagnosis.Definition> STAGNATING_ACTION_DEFINITIONS = RULES_BY_ACTION_CONFIG.entrySet()
+    static final Map<String, Diagnosis.Definition> STAGNATING_ACTION_DEFINITIONS = RULES_BY_ACTION_CONFIG.keySet()
         .stream()
         .collect(
             Collectors.toUnmodifiableMap(
-                Map.Entry::getKey,
-                entry -> new Diagnosis.Definition(
+                Function.identity(),
+                action -> new Diagnosis.Definition(
                     NAME,
-                    "stagnating_action:" + entry.getKey(),
-                    "Some indices have been stagnated on the action [" + entry.getKey() + "] longer than the expected time.",
+                    "stagnating_action:" + action,
+                    "Some indices have been stagnated on the action [" + action + "] longer than the expected time.",
                     "Check the current status of the Index Lifecycle Management for every affected index using the "
                         + "[GET /<affected_index_name>/_ilm/explain] API. Please replace the <affected_index_name> in the API "
                         + "with the actual index name.",
@@ -290,13 +319,23 @@ public class IlmHealthIndicatorService implements HealthIndicatorService {
      */
     static class StagnatingIndicesFinder {
         private final ClusterService clusterService;
-        private final Collection<RuleConfig> rules;
         private final LongSupplier nowSupplier;
+        private final Collection<RuleCreator> rulesCreators;
+        private volatile Collection<RuleConfig> rules;
 
-        StagnatingIndicesFinder(ClusterService clusterService, Collection<RuleConfig> rules, LongSupplier nowSupplier) {
+        StagnatingIndicesFinder(ClusterService clusterService, Collection<RuleCreator> rulesCreators, LongSupplier nowSupplier) {
             this.clusterService = clusterService;
-            this.rules = rules;
+            this.rulesCreators = rulesCreators;
             this.nowSupplier = nowSupplier;
+
+            var clusterSettings = this.clusterService.getClusterSettings();
+
+            clusterSettings.addSettingsUpdateConsumer(
+                this::recreateRules,
+                List.of(MAX_TIME_ON_ACTION_SETTING, MAX_TIME_ON_STEP_SETTING, MAX_RETRIES_PER_STEP_SETTING)
+            );
+
+            recreateRules(clusterService.getSettings());
         }
 
         /**
@@ -305,6 +344,7 @@ public class IlmHealthIndicatorService implements HealthIndicatorService {
         public List<IndexMetadata> find() {
             var metadata = clusterService.state().metadata();
             var now = nowSupplier.getAsLong();
+
             return metadata.indices()
                 .values()
                 .stream()
@@ -312,12 +352,33 @@ public class IlmHealthIndicatorService implements HealthIndicatorService {
                 .filter(md -> isStagnated(rules, now, md))
                 .toList();
         }
+
+        void recreateRules(Settings settings) {
+            var maxTimeOnAction = MAX_TIME_ON_ACTION_SETTING.get(settings);
+            var maxTimeOnStep = MAX_TIME_ON_STEP_SETTING.get(settings);
+            var maxRetriesPerStep = MAX_RETRIES_PER_STEP_SETTING.get(settings);
+
+            rules = rulesCreators.stream().map(rc -> rc.create(maxTimeOnAction, maxTimeOnStep, maxRetriesPerStep)).toList();
+        }
+
+        Collection<RuleConfig> rules() {
+            return rules;
+        }
+
+        ClusterService clusterService() {
+            return clusterService;
+        }
     }
 
     static boolean isStagnated(Collection<RuleConfig> rules, Long now, IndexMetadata indexMetadata) {
         return rules.stream().anyMatch(r -> r.test(now, indexMetadata));
     }
 
+    @FunctionalInterface
+    interface RuleCreator {
+        RuleConfig create(TimeValue defaultMaxTimeOnAction, TimeValue defaultMaxTimeOnStep, Long defaultMaxRetriesPerStep);
+    }
+
     @FunctionalInterface
     public interface RuleConfig {
 

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java

@@ -190,7 +190,10 @@ public class IndexLifecycle extends Plugin implements ActionPlugin, HealthPlugin
             LifecycleSettings.SLM_RETENTION_SCHEDULE_SETTING,
             LifecycleSettings.SLM_RETENTION_DURATION_SETTING,
             LifecycleSettings.SLM_MINIMUM_INTERVAL_SETTING,
-            LifecycleSettings.SLM_HEALTH_FAILED_SNAPSHOT_WARN_THRESHOLD_SETTING
+            LifecycleSettings.SLM_HEALTH_FAILED_SNAPSHOT_WARN_THRESHOLD_SETTING,
+            IlmHealthIndicatorService.MAX_TIME_ON_ACTION_SETTING,
+            IlmHealthIndicatorService.MAX_TIME_ON_STEP_SETTING,
+            IlmHealthIndicatorService.MAX_RETRIES_PER_STEP_SETTING
         );
     }
 
@@ -278,7 +281,7 @@ public class IndexLifecycle extends Plugin implements ActionPlugin, HealthPlugin
                 clusterService,
                 new IlmHealthIndicatorService.StagnatingIndicesFinder(
                     clusterService,
-                    IlmHealthIndicatorService.ILM_RULE_EVALUATOR,
+                    IlmHealthIndicatorService.RULES_BY_ACTION_CONFIG.values(),
                     System::currentTimeMillis
                 )
             )

x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IlmHealthIndicatorServiceTests.java

@@ -14,6 +14,8 @@ import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.health.Diagnosis;
 import org.elasticsearch.health.HealthIndicatorImpact;
@@ -37,6 +39,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import static org.elasticsearch.cluster.metadata.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
 import static org.elasticsearch.health.HealthStatus.GREEN;
@@ -322,6 +325,16 @@ public class IlmHealthIndicatorServiceTests extends ESTestCase {
     ) {
         var clusterService = mock(ClusterService.class);
         when(clusterService.state()).thenReturn(clusterState);
+        when(clusterService.getClusterSettings()).thenReturn(
+            new ClusterSettings(
+                Settings.EMPTY,
+                Set.of(
+                    IlmHealthIndicatorService.MAX_TIME_ON_ACTION_SETTING,
+                    IlmHealthIndicatorService.MAX_TIME_ON_STEP_SETTING,
+                    IlmHealthIndicatorService.MAX_RETRIES_PER_STEP_SETTING
+                )
+            )
+        );
 
         return new IlmHealthIndicatorService(clusterService, stagnatingIndicesFinder);
     }

x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/StagnatingIndicesFinderTests.java

@@ -13,6 +13,9 @@ import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.LifecycleExecutionState;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.ilm.LifecycleSettings;
 
@@ -20,12 +23,18 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.LongSupplier;
+import java.util.stream.Collectors;
 import java.util.stream.IntStream;
+import java.util.stream.Stream;
 
 import static org.elasticsearch.cluster.metadata.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
+import static org.elasticsearch.xpack.ilm.IlmHealthIndicatorService.MAX_RETRIES_PER_STEP_SETTING;
+import static org.elasticsearch.xpack.ilm.IlmHealthIndicatorService.MAX_TIME_ON_ACTION_SETTING;
+import static org.elasticsearch.xpack.ilm.IlmHealthIndicatorService.MAX_TIME_ON_STEP_SETTING;
 import static org.elasticsearch.xpack.ilm.IlmHealthIndicatorService.isStagnated;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.hasSize;
@@ -35,22 +44,33 @@ import static org.mockito.Mockito.when;
 public class StagnatingIndicesFinderTests extends ESTestCase {
 
     public void testStagnatingIndicesFinder() {
+        var maxTimeOnAction = randomTimeValueInDays();
+        var maxTimeOnStep = randomTimeValueInDays();
+        Long maxRetriesPerStep = randomLongBetween(2, 100);
         var idxMd1 = randomIndexMetadata();
         var idxMd2 = randomIndexMetadata();
         var idxMd3 = randomIndexMetadata();
         var stagnatingIndices = List.of(idxMd1.indexName, idxMd3.indexName);
         var mockedTimeSupplier = mock(LongSupplier.class);
         var instant = (long) randomIntBetween(100000, 200000);
-        var ruleEvaluator = List.<IlmHealthIndicatorService.RuleConfig>of(
+        var ruleCreator = Stream.<IlmHealthIndicatorService.RuleConfig>of(
             (now, indexMetadata) -> now == instant && stagnatingIndices.contains(indexMetadata.getIndex().getName())
-        );
+        ).map(rc -> (IlmHealthIndicatorService.RuleCreator) (expectedMaxTimeOnAction, expectedMaxTimeOnStep, expectedMaxRetriesPerStep) -> {
+            assertEquals(expectedMaxTimeOnAction, maxTimeOnAction);
+            assertEquals(expectedMaxTimeOnStep, maxTimeOnStep);
+            assertEquals(expectedMaxRetriesPerStep, maxRetriesPerStep);
+            return rc;
+        }).collect(Collectors.toList());
         // Per the evaluator, the timeSupplier _must_ be called only twice
         when(mockedTimeSupplier.getAsLong()).thenReturn(instant, instant);
 
         var stagnatedIdx1 = indexMetadataFrom(idxMd1);
         var stagnatedIdx3 = indexMetadataFrom(idxMd3);
         var finder = createStagnatingIndicesFinder(
-            ruleEvaluator,
+            ruleCreator,
+            maxTimeOnAction,
+            maxTimeOnStep,
+            maxRetriesPerStep,
             mockedTimeSupplier,
             indexMetadataUnmanaged(randomAlphaOfLength(10)), // non-managed by ILM
             stagnatedIdx1,                                 // should be stagnated
@@ -65,6 +85,96 @@ public class StagnatingIndicesFinderTests extends ESTestCase {
         assertThat(foundIndices, containsInAnyOrder(stagnatedIdx1, stagnatedIdx3));
     }
 
+    public void testRecreateRules() {
+        var maxTimeOnAction = randomTimeValueInDays();
+        var maxTimeOnStep = randomTimeValueInDays();
+        long maxRetriesPerStep = randomLongBetween(2, 100);
+        var accumulator = new ArrayList<Integer>();
+        var numberOfRuleConfigs = randomIntBetween(3, 20);
+
+        var ruleCreators = IntStream.range(0, numberOfRuleConfigs)
+            .mapToObj(
+                i -> (IlmHealthIndicatorService.RuleCreator) (
+                    expectedMaxTimeOnAction,
+                    expectedMaxTimeOnStep,
+                    expectedMaxRetriesPerStep) -> {
+                    accumulator.add(i);
+                    return new RuleConfigHolder(expectedMaxTimeOnAction, expectedMaxTimeOnStep, expectedMaxRetriesPerStep);
+                }
+            )
+            .toList();
+
+        var finder = createStagnatingIndicesFinder(
+            ruleCreators,
+            maxTimeOnAction,
+            maxTimeOnStep,
+            maxRetriesPerStep,
+            mock(LongSupplier.class)
+        );
+
+        // Test: safety-net ensuring that the settings are gathered when the object is created
+        assertEquals(accumulator, IntStream.range(0, numberOfRuleConfigs).boxed().toList());
+
+        var rules = finder.rules();
+        assertThat(rules, hasSize(numberOfRuleConfigs));
+        // all the rules will have the same value, so it's enough checking the first one
+        assertEquals(rules.iterator().next(), new RuleConfigHolder(maxTimeOnAction, maxTimeOnStep, maxRetriesPerStep));
+
+        accumulator.clear();
+        maxTimeOnAction = randomTimeValueInDays();
+        maxTimeOnStep = randomTimeValueInDays();
+        maxRetriesPerStep = randomLongBetween(2, 100);
+
+        // Test: the method `recreateRules` works as expected
+        finder.recreateRules(
+            Settings.builder()
+                .put(MAX_TIME_ON_ACTION_SETTING.getKey(), maxTimeOnAction)
+                .put(MAX_TIME_ON_STEP_SETTING.getKey(), maxTimeOnStep)
+                .put(MAX_RETRIES_PER_STEP_SETTING.getKey(), maxRetriesPerStep)
+                .build()
+        );
+
+        var newRules = finder.rules();
+        assertThat(rules, hasSize(numberOfRuleConfigs));
+        assertNotSame(rules, newRules);
+        // all the rules will have the same value, so it's enough checking the first one
+        assertEquals(newRules.iterator().next(), new RuleConfigHolder(maxTimeOnAction, maxTimeOnStep, maxRetriesPerStep));
+        assertEquals(accumulator, IntStream.range(0, numberOfRuleConfigs).boxed().toList());
+
+        accumulator.clear();
+        rules = finder.rules();
+        maxTimeOnAction = randomTimeValueInDays();
+        maxTimeOnStep = randomTimeValueInDays();
+        maxRetriesPerStep = randomLongBetween(2, 100);
+
+        // Test: Force a settings update, ensuring that the method `recreateRules` is called
+        finder.clusterService()
+            .getClusterSettings()
+            .applySettings(
+                Settings.builder()
+                    .put(MAX_TIME_ON_ACTION_SETTING.getKey(), maxTimeOnAction)
+                    .put(MAX_TIME_ON_STEP_SETTING.getKey(), maxTimeOnStep)
+                    .put(MAX_RETRIES_PER_STEP_SETTING.getKey(), maxRetriesPerStep)
+                    .build()
+            );
+
+        newRules = finder.rules();
+        assertThat(rules, hasSize(numberOfRuleConfigs));
+        assertNotSame(rules, newRules);
+        // all the rules will have the same value, so it's enough checking the first one
+        assertEquals(newRules.iterator().next(), new RuleConfigHolder(maxTimeOnAction, maxTimeOnStep, maxRetriesPerStep));
+        assertEquals(accumulator, IntStream.range(0, numberOfRuleConfigs).boxed().toList());
+    }
+
+    record RuleConfigHolder(TimeValue maxTimeOnAction, TimeValue maxTimeOnStep, Long maxRetries)
+        implements
+            IlmHealthIndicatorService.RuleConfig {
+        @Override
+        public boolean test(Long now, IndexMetadata indexMetadata) {
+            return false;
+        }
+    }
+
     public void testStagnatingIndicesEvaluator() {
         var idxMd1 = randomIndexMetadata();
         var indexMetadata = indexMetadataFrom(idxMd1);
@@ -111,6 +221,10 @@ public class StagnatingIndicesFinderTests extends ESTestCase {
         }
     }
 
+    private static TimeValue randomTimeValueInDays() {
+        return TimeValue.parseTimeValue(randomTimeValue(1, 1000, "d"), "some.name");
+    }
+
     private static IndexMetadata indexMetadataUnmanaged(String indexName) {
         return indexMetadataFrom(new IndexMetadataTestCase(indexName, null, null));
     }
@@ -131,7 +245,10 @@ public class StagnatingIndicesFinderTests extends ESTestCase {
     }
 
     private IlmHealthIndicatorService.StagnatingIndicesFinder createStagnatingIndicesFinder(
-        Collection<IlmHealthIndicatorService.RuleConfig> evaluator,
+        Collection<IlmHealthIndicatorService.RuleCreator> ruleCreator,
+        TimeValue maxTimeOnAction,
+        TimeValue maxTimeOnStep,
+        long maxRetriesPerStep,
         LongSupplier timeSupplier,
         IndexMetadata... indicesMetadata
     ) {
@@ -143,8 +260,24 @@ public class StagnatingIndicesFinderTests extends ESTestCase {
         when(state.metadata()).thenReturn(metadataBuilder.build());
 
         when(clusterService.state()).thenReturn(state);
+        var settings = Settings.builder()
+            .put(MAX_TIME_ON_ACTION_SETTING.getKey(), maxTimeOnAction)
+            .put(MAX_TIME_ON_STEP_SETTING.getKey(), maxTimeOnStep)
+            .put(MAX_RETRIES_PER_STEP_SETTING.getKey(), maxRetriesPerStep)
+            .build();
+        when(clusterService.getSettings()).thenReturn(settings);
+        when(clusterService.getClusterSettings()).thenReturn(
+            new ClusterSettings(
+                settings,
+                Set.of(
+                    IlmHealthIndicatorService.MAX_TIME_ON_ACTION_SETTING,
+                    IlmHealthIndicatorService.MAX_TIME_ON_STEP_SETTING,
+                    IlmHealthIndicatorService.MAX_RETRIES_PER_STEP_SETTING
+                )
+            )
+        );
 
-        return new IlmHealthIndicatorService.StagnatingIndicesFinder(clusterService, evaluator, timeSupplier);
+        return new IlmHealthIndicatorService.StagnatingIndicesFinder(clusterService, ruleCreator, timeSupplier);
     }
 
     static IndexMetadataTestCase randomIndexMetadata() {