Add extension points to remediate index metadata in during snapshot restore (#131706)

Adds a new SPI to allow plugins to hook into the snapshot restore process in order to remediate 
index metadata before accepting it into the cluster state.
---------
Co-authored-by: elasticsearchmachine <infra-root+elasticsearchmachine@elastic.co>

docs/changelog/131706.yaml

@@ -0,0 +1,5 @@
+pr: 131706
+summary: Add extension points to remediate index metadata in during snapshot restore
+area: Snapshot/Restore
+type: enhancement
+issues: []

server/src/internalClusterTest/java/org/elasticsearch/snapshots/RemediateSnapshotIT.java

@@ -0,0 +1,109 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.snapshots;
+
+import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
+import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.plugins.Plugin;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.stream.Stream;
+
+import static org.elasticsearch.index.IndexSettings.INDEX_SEARCH_IDLE_AFTER;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+
+public class RemediateSnapshotIT extends AbstractSnapshotIntegTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Stream.concat(Stream.of(RemediateSnapshotTestPlugin.class), super.nodePlugins().stream()).toList();
+    }
+
+    public void testRemediationOnRestore() {
+        Client client = client();
+
+        createRepository("test-repo", "fs");
+
+        createIndex("test-idx-1", "test-idx-2", "test-idx-3");
+        ensureGreen();
+
+        GetIndexResponse getIndexResponse = client.admin()
+            .indices()
+            .prepareGetIndex(TEST_REQUEST_TIMEOUT)
+            .setIndices("test-idx-1", "test-idx-2", "test-idx-3")
+            .get();
+
+        assertThat(
+            INDEX_SEARCH_IDLE_AFTER.get(getIndexResponse.settings().get("test-idx-1")),
+            equalTo(INDEX_SEARCH_IDLE_AFTER.getDefault(Settings.EMPTY))
+        );
+        assertThat(
+            INDEX_SEARCH_IDLE_AFTER.get(getIndexResponse.settings().get("test-idx-2")),
+            equalTo(INDEX_SEARCH_IDLE_AFTER.getDefault(Settings.EMPTY))
+        );
+        assertThat(
+            INDEX_SEARCH_IDLE_AFTER.get(getIndexResponse.settings().get("test-idx-3")),
+            equalTo(INDEX_SEARCH_IDLE_AFTER.getDefault(Settings.EMPTY))
+        );
+
+        indexRandomDocs("test-idx-1", 100);
+        indexRandomDocs("test-idx-2", 100);
+
+        createSnapshot("test-repo", "test-snap", Arrays.asList("test-idx-1", "test-idx-2"));
+
+        logger.info("--> close snapshot indices");
+        client.admin().indices().prepareClose("test-idx-1", "test-idx-2").get();
+
+        logger.info("--> restore indices");
+        RestoreSnapshotResponse restoreSnapshotResponse = client.admin()
+            .cluster()
+            .prepareRestoreSnapshot(TEST_REQUEST_TIMEOUT, "test-repo", "test-snap")
+            .setWaitForCompletion(true)
+            .get();
+        assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
+
+        GetIndexResponse getIndexResponseAfter = client.admin()
+            .indices()
+            .prepareGetIndex(TEST_REQUEST_TIMEOUT)
+            .setIndices("test-idx-1", "test-idx-2", "test-idx-3")
+            .get();
+
+        assertDocCount("test-idx-1", 100L);
+        assertThat(INDEX_SEARCH_IDLE_AFTER.get(getIndexResponseAfter.settings().get("test-idx-1")), equalTo(TimeValue.timeValueMinutes(2)));
+        assertDocCount("test-idx-2", 100L);
+        assertThat(INDEX_SEARCH_IDLE_AFTER.get(getIndexResponseAfter.settings().get("test-idx-2")), equalTo(TimeValue.timeValueMinutes(2)));
+    }
+
+    /**
+     * Dummy plugin to load SPI function off of
+     */
+    public static class RemediateSnapshotTestPlugin extends Plugin {}
+
+    public static class RemediateSnapshotTestTransformer implements IndexMetadataRestoreTransformer {
+        @Override
+        public IndexMetadata updateIndexMetadata(IndexMetadata original) {
+            // Set a property to something mild, outside its default
+            return IndexMetadata.builder(original)
+                .settings(
+                    Settings.builder()
+                        .put(original.getSettings())
+                        .put(INDEX_SEARCH_IDLE_AFTER.getKey(), TimeValue.timeValueMinutes(2))
+                        .build()
+                )
+                .build();
+        }
+    }
+}

server/src/internalClusterTest/resources/META-INF/services/org.elasticsearch.snapshots.IndexMetadataRestoreTransformer

@@ -0,0 +1,10 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the "Elastic License
+# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+# Public License v 1"; you may not use this file except in compliance with, at
+# your election, the "Elastic License 2.0", the "GNU Affero General Public
+# License v3.0 only", or the "Server Side Public License, v 1".
+#
+
+org.elasticsearch.snapshots.RemediateSnapshotIT$RemediateSnapshotTestTransformer

server/src/main/java/org/elasticsearch/node/NodeConstruction.java

@@ -205,6 +205,8 @@ import org.elasticsearch.search.SearchService;
 import org.elasticsearch.search.SearchUtils;
 import org.elasticsearch.search.aggregations.support.AggregationUsageService;
 import org.elasticsearch.shutdown.PluginShutdownService;
+import org.elasticsearch.snapshots.IndexMetadataRestoreTransformer;
+import org.elasticsearch.snapshots.IndexMetadataRestoreTransformer.NoOpRestoreTransformer;
 import org.elasticsearch.snapshots.InternalSnapshotsInfoService;
 import org.elasticsearch.snapshots.RepositoryIntegrityHealthIndicatorService;
 import org.elasticsearch.snapshots.RestoreService;
@@ -1160,7 +1162,8 @@ class NodeConstruction {
             indicesService,
             fileSettingsService,
             threadPool,
-            projectResolver.supportsMultipleProjects()
+            projectResolver.supportsMultipleProjects(),
+            pluginsService.loadSingletonServiceProvider(IndexMetadataRestoreTransformer.class, NoOpRestoreTransformer::getInstance)
         );
 
         DiscoveryModule discoveryModule = createDiscoveryModule(

server/src/main/java/org/elasticsearch/snapshots/IndexMetadataRestoreTransformer.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.snapshots;
+
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+
+/**
+ * A temporary interface meant to allow a plugin to provide remedial logic for index metadata being restored from a snapshot. This was
+ * added to address an allocation issue and will eventually be removed once any affected snapshots age out.
+ */
+public interface IndexMetadataRestoreTransformer {
+    IndexMetadata updateIndexMetadata(IndexMetadata original);
+
+    /**
+     * A default implementation of {@link IndexMetadataRestoreTransformer} which does nothing
+     */
+    final class NoOpRestoreTransformer implements IndexMetadataRestoreTransformer {
+        public static final NoOpRestoreTransformer INSTANCE = new NoOpRestoreTransformer();
+
+        public static NoOpRestoreTransformer getInstance() {
+            return INSTANCE;
+        }
+
+        private NoOpRestoreTransformer() {}
+
+        @Override
+        public IndexMetadata updateIndexMetadata(IndexMetadata original) {
+            return original;
+        }
+    }
+}

server/src/main/java/org/elasticsearch/snapshots/RestoreService.java

@@ -206,6 +206,8 @@ public final class RestoreService implements ClusterStateApplier {
 
     private final Executor snapshotMetaExecutor;
 
+    private final IndexMetadataRestoreTransformer indexMetadataRestoreTransformer;
+
     private volatile boolean refreshRepositoryUuidOnRestore;
 
     public RestoreService(
@@ -219,7 +221,8 @@ public final class RestoreService implements ClusterStateApplier {
         IndicesService indicesService,
         FileSettingsService fileSettingsService,
         ThreadPool threadPool,
-        boolean deserializeProjectMetadata
+        boolean deserializeProjectMetadata,
+        IndexMetadataRestoreTransformer indexMetadataRestoreTransformer
     ) {
         this.clusterService = clusterService;
         this.repositoriesService = repositoriesService;
@@ -240,6 +243,7 @@ public final class RestoreService implements ClusterStateApplier {
         this.refreshRepositoryUuidOnRestore = REFRESH_REPO_UUID_ON_RESTORE_SETTING.get(clusterService.getSettings());
         clusterService.getClusterSettings()
             .addSettingsUpdateConsumer(REFRESH_REPO_UUID_ON_RESTORE_SETTING, this::setRefreshRepositoryUuidOnRestore);
+        this.indexMetadataRestoreTransformer = indexMetadataRestoreTransformer;
     }
 
     /**
@@ -521,6 +525,8 @@ public final class RestoreService implements ClusterStateApplier {
         }
         for (IndexId indexId : repositoryData.resolveIndices(requestedIndicesIncludingSystem).values()) {
             IndexMetadata snapshotIndexMetaData = repository.getSnapshotIndexMetaData(repositoryData, snapshotId, indexId);
+            // Update the snapshot index metadata before adding it to the metadata
+            snapshotIndexMetaData = indexMetadataRestoreTransformer.updateIndexMetadata(snapshotIndexMetaData);
             if (snapshotIndexMetaData.isSystem()) {
                 if (requestIndices.contains(indexId.getName())) {
                     explicitlyRequestedSystemIndices.add(indexId.getName());

server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java

@@ -2703,7 +2703,8 @@ public class SnapshotResiliencyTests extends ESTestCase {
                     indicesService,
                     mock(FileSettingsService.class),
                     threadPool,
-                    false
+                    false,
+                    IndexMetadataRestoreTransformer.NoOpRestoreTransformer.getInstance()
                 );
                 actions.put(
                     TransportPutMappingAction.TYPE,