Fix ILM history index settings (#61880)

This change fixed settings accidentally changed in #61457, this fixes ability of ILM history index to rollover.
It also adds test to make sure we can't hit this kind of bug in the future.

Closes #61863

x-pack/plugin/core/src/main/resources/ilm-history.json

@@ -7,7 +7,10 @@
       "index.number_of_shards": 1,
       "index.number_of_replicas": 0,
       "index.auto_expand_replicas": "0-1",
-      "index.lifecycle.name": "ilm-history-ilm-policy"
+      "index.lifecycle.name": "ilm-history-ilm-policy",
+      "index.lifecycle.rollover_alias": "ilm-history-${xpack.ilm_history.template.version}",
+      "index.hidden": true,
+      "index.format": 1
     },
     "mappings": {
         "dynamic": false,

x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMHistoryTests.java

@@ -0,0 +1,113 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.ilm;
+
+import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
+import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
+import org.elasticsearch.action.admin.indices.rollover.RolloverResponse;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.ilm.LifecyclePolicy;
+import org.elasticsearch.xpack.core.ilm.LifecycleSettings;
+import org.elasticsearch.xpack.core.ilm.OperationMode;
+import org.elasticsearch.xpack.core.ilm.Phase;
+import org.elasticsearch.xpack.core.ilm.StopILMRequest;
+import org.elasticsearch.xpack.core.ilm.action.GetStatusAction;
+import org.elasticsearch.xpack.core.ilm.action.PutLifecycleAction;
+import org.elasticsearch.xpack.core.ilm.action.StopILMAction;
+import org.elasticsearch.xpack.ilm.history.ILMHistoryStore;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+
+import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
+import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;
+import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.arrayContaining;
+import static org.hamcrest.Matchers.is;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 1)
+public class ILMHistoryTests extends ESIntegTestCase {
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        Settings.Builder settings = Settings.builder().put(super.nodeSettings(nodeOrdinal));
+        settings.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false);
+        settings.put(XPackSettings.SECURITY_ENABLED.getKey(), false);
+        settings.put(XPackSettings.WATCHER_ENABLED.getKey(), false);
+        settings.put(XPackSettings.GRAPH_ENABLED.getKey(), false);
+        settings.put(LifecycleSettings.LIFECYCLE_POLL_INTERVAL, "1s");
+        settings.put(LifecycleSettings.SLM_HISTORY_INDEX_ENABLED_SETTING.getKey(), false);
+        return settings.build();
+    }
+
+    @Override
+    protected boolean ignoreExternalCluster() {
+        return true;
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Arrays.asList(LocalStateCompositeXPackPlugin.class, IndexLifecycle.class);
+    }
+
+    private void putTestPolicy() throws InterruptedException, java.util.concurrent.ExecutionException {
+        Phase phase = new Phase("hot", TimeValue.ZERO, Collections.emptyMap());
+        LifecyclePolicy lifecyclePolicy = new LifecyclePolicy("test", Collections.singletonMap("hot", phase));
+        PutLifecycleAction.Request putLifecycleRequest = new PutLifecycleAction.Request(lifecyclePolicy);
+        PutLifecycleAction.Response putLifecycleResponse = client().execute(PutLifecycleAction.INSTANCE, putLifecycleRequest).get();
+        assertAcked(putLifecycleResponse);
+    }
+
+    public void testIlmHistoryIndexCanRollover() throws Exception {
+        putTestPolicy();
+        Settings settings = Settings.builder().put(indexSettings()).put(SETTING_NUMBER_OF_SHARDS, 1)
+            .put(SETTING_NUMBER_OF_REPLICAS, 0).put(LifecycleSettings.LIFECYCLE_NAME, "test").build();
+        CreateIndexResponse res = client().admin().indices().prepareCreate("test").setSettings(settings).get();
+        assertTrue(res.isAcknowledged());
+
+        String firstIndex = ILMHistoryStore.ILM_HISTORY_INDEX_PREFIX + "000001";
+        String secondIndex = ILMHistoryStore.ILM_HISTORY_INDEX_PREFIX + "000002";
+
+        assertBusy(() -> {
+            try {
+                GetIndexResponse getIndexResponse = client().admin().indices().prepareGetIndex().setIndices(firstIndex).get();
+                assertThat(getIndexResponse.getIndices(), arrayContaining(firstIndex));
+            } catch (Exception e) {
+                fail(e.getMessage());
+            }
+        });
+
+        //wait for all history items to index to avoid waiting for timeout in ILMHistoryStore beforeBulk
+        assertBusy(() -> {
+            SearchResponse search = client().prepareSearch(firstIndex).setQuery(matchQuery("index", firstIndex)).setSize(0).get();
+            assertThat(search.getHits().getTotalHits().value, is(9L));
+        });
+
+        //make sure ILM is stopped so no new items will be queued in ILM history
+        assertTrue(client().execute(StopILMAction.INSTANCE, new StopILMRequest()).actionGet().isAcknowledged());
+        assertBusy(() -> {
+            GetStatusAction.Response status = client().execute(GetStatusAction.INSTANCE, new GetStatusAction.Request()).actionGet();
+            assertThat(status.getMode(), is(OperationMode.STOPPED));
+        });
+
+        RolloverResponse rolloverResponse = client().admin().indices().prepareRolloverIndex(ILMHistoryStore.ILM_HISTORY_ALIAS).get();
+
+        assertTrue(rolloverResponse.isAcknowledged());
+        assertThat(rolloverResponse.getNewIndex(), is(secondIndex));
+
+        GetIndexResponse getIndexResponse = client().admin().indices().prepareGetIndex().setIndices(secondIndex).get();
+        assertThat(getIndexResponse.getIndices(), arrayContaining(secondIndex));
+    }
+}