Recover broken IndexMetaData as closed

Today if something is wrong with the IndexMetaData we detect it very
late and most of the time if that happens we already allocated the index
and get endless loops and full log files on data-nodes. This change tries
to verify IndexService creattion during initial state recovery on the master
and if the recovery fails the index is imported as `closed` and won't be allocated
at all.

Closes #17187

buildSrc/src/main/resources/checkstyle_suppressions.xml

@@ -419,7 +419,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]env[/\\]NodeEnvironment.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]AsyncShardFetch.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]DanglingIndicesState.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]Gateway.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]GatewayAllocator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]GatewayMetaState.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]GatewayService.java" checks="LineLength" />

core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateService.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.cluster.metadata;
 
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.close.CloseIndexClusterStateUpdateRequest;
 import org.elasticsearch.action.admin.indices.open.OpenIndexClusterStateUpdateRequest;
@@ -37,6 +38,8 @@ import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.index.NodeServicesProvider;
+import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.snapshots.RestoreService;
 import org.elasticsearch.snapshots.SnapshotsService;
@@ -59,10 +62,16 @@ public class MetaDataIndexStateService extends AbstractComponent {
     private final AllocationService allocationService;
 
     private final MetaDataIndexUpgradeService metaDataIndexUpgradeService;
+    private final NodeServicesProvider nodeServiceProvider;
+    private final IndicesService indicesService;
 
     @Inject
-    public MetaDataIndexStateService(Settings settings, ClusterService clusterService, AllocationService allocationService, MetaDataIndexUpgradeService metaDataIndexUpgradeService) {
+    public MetaDataIndexStateService(Settings settings, ClusterService clusterService, AllocationService allocationService,
+                                     MetaDataIndexUpgradeService metaDataIndexUpgradeService,
+                                     NodeServicesProvider nodeServicesProvider, IndicesService indicesService) {
         super(settings);
+        this.nodeServiceProvider = nodeServicesProvider;
+        this.indicesService = indicesService;
         this.clusterService = clusterService;
         this.allocationService = allocationService;
         this.metaDataIndexUpgradeService = metaDataIndexUpgradeService;
@@ -162,6 +171,12 @@ public class MetaDataIndexStateService extends AbstractComponent {
                     // The index might be closed because we couldn't import it due to old incompatible version
                     // We need to check that this index can be upgraded to the current version
                     indexMetaData = metaDataIndexUpgradeService.upgradeIndexMetaData(indexMetaData);
+                    try {
+                        indicesService.verifyIndexMetadata(nodeServiceProvider, indexMetaData);
+                    } catch (Exception e) {
+                        throw new ElasticsearchException("Failed to verify index " + indexMetaData.getIndex(), e);
+                    }
+
                     mdBuilder.put(indexMetaData, true);
                     blocksBuilder.removeIndexBlock(indexName, INDEX_CLOSED_BLOCK);
                 }

core/src/main/java/org/elasticsearch/gateway/Gateway.java

@@ -35,8 +35,14 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.discovery.Discovery;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.NodeServicesProvider;
+import org.elasticsearch.indices.IndicesService;
 
+import java.io.IOException;
 import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.function.Supplier;
 
 /**
@@ -53,10 +59,15 @@ public class Gateway extends AbstractComponent implements ClusterStateListener {
     private final TransportNodesListGatewayMetaState listGatewayMetaState;
 
     private final Supplier<Integer> minimumMasterNodesProvider;
+    private final IndicesService indicesService;
+    private final NodeServicesProvider nodeServicesProvider;
 
     public Gateway(Settings settings, ClusterService clusterService, NodeEnvironment nodeEnv, GatewayMetaState metaState,
-                   TransportNodesListGatewayMetaState listGatewayMetaState, Discovery discovery) {
+                   TransportNodesListGatewayMetaState listGatewayMetaState, Discovery discovery,
+                   NodeServicesProvider nodeServicesProvider, IndicesService indicesService) {
         super(settings);
+        this.nodeServicesProvider = nodeServicesProvider;
+        this.indicesService = indicesService;
         this.clusterService = clusterService;
         this.nodeEnv = nodeEnv;
         this.metaState = metaState;
@@ -66,9 +77,9 @@ public class Gateway extends AbstractComponent implements ClusterStateListener {
     }
 
     public void performStateRecovery(final GatewayStateRecoveredListener listener) throws GatewayException {
-        ObjectHashSet<String> nodesIds = new ObjectHashSet<>(clusterService.state().nodes().masterNodes().keys());
-        logger.trace("performing state recovery from {}", nodesIds);
-        TransportNodesListGatewayMetaState.NodesGatewayMetaState nodesState = listGatewayMetaState.list(nodesIds.toArray(String.class), null).actionGet();
+        String[] nodesIds = clusterService.state().nodes().masterNodes().keys().toArray(String.class);
+        logger.trace("performing state recovery from {}", Arrays.toString(nodesIds));
+        TransportNodesListGatewayMetaState.NodesGatewayMetaState nodesState = listGatewayMetaState.list(nodesIds, null).actionGet();
 
 
         int requiredAllocation = Math.max(1, minimumMasterNodesProvider.get());
@@ -129,7 +140,17 @@ public class Gateway extends AbstractComponent implements ClusterStateListener {
                 if (electedIndexMetaData != null) {
                     if (indexMetaDataCount < requiredAllocation) {
                         logger.debug("[{}] found [{}], required [{}], not adding", index, indexMetaDataCount, requiredAllocation);
+                    } // TODO if this logging statement is correct then we are missing an else here
+                    try {
+                        if (electedIndexMetaData.getState() == IndexMetaData.State.OPEN) {
+                            // verify that we can actually create this index - if not we recover it as closed with lots of warn logs
+                            indicesService.verifyIndexMetadata(nodeServicesProvider, electedIndexMetaData);
+                        }
+                    } catch (Exception e) {
+                        logger.warn("recovering index {} failed - recovering as closed", e, electedIndexMetaData.getIndex());
+                        electedIndexMetaData = IndexMetaData.builder(electedIndexMetaData).state(IndexMetaData.State.CLOSE).build();
                     }
+
                     metaDataBuilder.put(electedIndexMetaData, false);
                 }
             }

core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java

@@ -86,8 +86,8 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateL
         if (DiscoveryNode.masterNode(settings) || DiscoveryNode.dataNode(settings)) {
             try {
                 ensureNoPre019State();
-                pre20Upgrade();
                 IndexFolderUpgrader.upgradeIndicesIfNeeded(settings, nodeEnv);
+                upgradeMetaData();
                 long startNS = System.nanoTime();
                 metaStateService.loadFullState();
                 logger.debug("took {} to load state", TimeValue.timeValueMillis(TimeValue.nsecToMSec(System.nanoTime() - startNS)));
@@ -222,7 +222,7 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateL
      * MetaDataIndexUpgradeService might also update obsolete settings if needed. When this happens we rewrite
      * index metadata with new settings.
      */
-    private void pre20Upgrade() throws Exception {
+    private void upgradeMetaData() throws Exception {
         MetaData metaData = loadMetaState();
         List<IndexMetaData> updateIndexMetaData = new ArrayList<>();
         for (IndexMetaData indexMetaData : metaData) {
@@ -235,7 +235,7 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateL
         // means the upgrade can continue. Now it's safe to overwrite index metadata with the new version.
         for (IndexMetaData indexMetaData : updateIndexMetaData) {
             // since we still haven't upgraded the index folders, we write index state in the old folder
-            metaStateService.writeIndex("upgrade", indexMetaData, nodeEnv.resolveIndexFolder(indexMetaData.getIndex().getName()));
+            metaStateService.writeIndex("upgrade", indexMetaData, nodeEnv.resolveIndexFolder(indexMetaData.getIndex().getUUID()));
         }
     }
 

core/src/main/java/org/elasticsearch/gateway/GatewayService.java

@@ -43,6 +43,8 @@ import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.discovery.Discovery;
 import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.index.NodeServicesProvider;
+import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.threadpool.ThreadPool;
 
@@ -95,9 +97,11 @@ public class GatewayService extends AbstractLifecycleComponent<GatewayService> i
     @Inject
     public GatewayService(Settings settings, AllocationService allocationService, ClusterService clusterService,
                           ThreadPool threadPool, NodeEnvironment nodeEnvironment, GatewayMetaState metaState,
-                          TransportNodesListGatewayMetaState listGatewayMetaState, Discovery discovery) {
+                          TransportNodesListGatewayMetaState listGatewayMetaState, Discovery discovery,
+                          NodeServicesProvider nodeServicesProvider, IndicesService indicesService) {
         super(settings);
-        this.gateway = new Gateway(settings, clusterService, nodeEnvironment, metaState, listGatewayMetaState, discovery);
+        this.gateway = new Gateway(settings, clusterService, nodeEnvironment, metaState, listGatewayMetaState, discovery,
+            nodeServicesProvider, indicesService);
         this.allocationService = allocationService;
         this.clusterService = clusterService;
         this.threadPool = threadPool;

core/src/main/java/org/elasticsearch/indices/IndicesService.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.indices;
 
+import com.carrotsearch.hppc.cursors.ObjectCursor;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.store.LockObtainFailedException;
 import org.apache.lucene.util.CollectionUtil;
@@ -33,6 +34,7 @@ import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.breaker.CircuitBreaker;
@@ -66,6 +68,7 @@ import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.flush.FlushStats;
 import org.elasticsearch.index.get.GetStats;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.merge.MergeStats;
 import org.elasticsearch.index.recovery.RecoveryStats;
 import org.elasticsearch.index.refresh.RefreshStats;
@@ -74,6 +77,7 @@ import org.elasticsearch.index.shard.IllegalIndexShardStateException;
 import org.elasticsearch.index.shard.IndexEventListener;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardState;
+import org.elasticsearch.index.shard.IndexingOperationListener;
 import org.elasticsearch.index.shard.IndexingStats;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.store.IndexStoreConfig;
@@ -88,9 +92,11 @@ import org.elasticsearch.search.query.QueryPhase;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.threadpool.ThreadPool;
 
+import java.io.Closeable;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -324,6 +330,7 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
      * @throws IndexAlreadyExistsException if the index already exists.
      */
     public synchronized IndexService createIndex(final NodeServicesProvider nodeServicesProvider, IndexMetaData indexMetaData, List<IndexEventListener> builtInListeners) throws IOException {
+
         if (!lifecycle.started()) {
             throw new IllegalStateException("Can't create an index [" + indexMetaData.getIndex() + "], node is closed");
         }
@@ -331,37 +338,22 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
             throw new IllegalArgumentException("index must have a real UUID found value: [" + indexMetaData.getIndexUUID() + "]");
         }
         final Index index = indexMetaData.getIndex();
-        final Predicate<String> indexNameMatcher = (indexExpression) -> indexNameExpressionResolver.matchesIndex(index.getName(), indexExpression, clusterService.state());
-        final IndexSettings idxSettings = new IndexSettings(indexMetaData, this.settings, indexNameMatcher, indexScopeSetting);
         if (hasIndex(index)) {
             throw new IndexAlreadyExistsException(index);
         }
-        logger.debug("creating Index [{}], shards [{}]/[{}{}]",
-                indexMetaData.getIndex(),
-                idxSettings.getNumberOfShards(),
-                idxSettings.getNumberOfReplicas(),
-                idxSettings.isShadowReplicaIndex() ? "s" : "");
-
-        final IndexModule indexModule = new IndexModule(idxSettings, indexStoreConfig, analysisRegistry);
-        pluginsService.onIndexModule(indexModule);
-        for (IndexEventListener listener : builtInListeners) {
-            indexModule.addIndexEventListener(listener);
-        }
+        List<IndexEventListener> finalListeners = new ArrayList<>(builtInListeners);
         final IndexEventListener onStoreClose = new IndexEventListener() {
             @Override
             public void onStoreClosed(ShardId shardId) {
                 indicesQueryCache.onClose(shardId);
             }
         };
-        indexModule.addIndexEventListener(onStoreClose);
-        indexModule.addIndexEventListener(oldShardsStats);
-        final IndexEventListener listener = indexModule.freeze();
-        listener.beforeIndexCreated(index, idxSettings.getSettings());
-        final IndexService indexService = indexModule.newIndexService(nodeEnv, this, nodeServicesProvider, indicesQueryCache, mapperRegistry, indicesFieldDataCache, indexingMemoryController);
+        finalListeners.add(onStoreClose);
+        finalListeners.add(oldShardsStats);
+        final IndexService indexService = createIndexService("create index", nodeServicesProvider, indexMetaData, indicesQueryCache, indicesFieldDataCache, finalListeners, indexingMemoryController);
         boolean success = false;
         try {
-            assert indexService.getIndexEventListener() == listener;
-            listener.afterIndexCreated(indexService);
+            indexService.getIndexEventListener().afterIndexCreated(indexService);
             indices = newMapBuilder(indices).put(index.getUUID(), indexService).immutableMap();
             success = true;
             return indexService;
@@ -370,7 +362,54 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
                 indexService.close("plugins_failed", true);
             }
         }
+    }
 
+    /**
+     * This creates a new IndexService without registering it
+     */
+    private synchronized IndexService createIndexService(final String reason, final NodeServicesProvider nodeServicesProvider, IndexMetaData indexMetaData, IndicesQueryCache indicesQueryCache, IndicesFieldDataCache indicesFieldDataCache, List<IndexEventListener> builtInListeners, IndexingOperationListener... indexingOperationListeners) throws IOException {
+        final Index index = indexMetaData.getIndex();
+        final Predicate<String> indexNameMatcher = (indexExpression) -> indexNameExpressionResolver.matchesIndex(index.getName(), indexExpression, clusterService.state());
+        final IndexSettings idxSettings = new IndexSettings(indexMetaData, this.settings, indexNameMatcher, indexScopeSetting);
+        logger.debug("creating Index [{}], shards [{}]/[{}{}] - reason [{}]",
+            indexMetaData.getIndex(),
+            idxSettings.getNumberOfShards(),
+            idxSettings.getNumberOfReplicas(),
+            idxSettings.isShadowReplicaIndex() ? "s" : "", reason);
+
+        final IndexModule indexModule = new IndexModule(idxSettings, indexStoreConfig, analysisRegistry);
+        pluginsService.onIndexModule(indexModule);
+        for (IndexEventListener listener : builtInListeners) {
+            indexModule.addIndexEventListener(listener);
+        }
+        final IndexEventListener listener = indexModule.freeze();
+        listener.beforeIndexCreated(index, idxSettings.getSettings());
+        return indexModule.newIndexService(nodeEnv, this, nodeServicesProvider, indicesQueryCache, mapperRegistry, indicesFieldDataCache, indexingOperationListeners);
+    }
+
+    /**
+     * This method verifies that the given {@link IndexMetaData} holds sane values to create an {@link IndexService}. This method will throw an
+     * exception if the creation fails. The created {@link IndexService} will not be registered and will be closed immediately.
+     */
+    public synchronized void verifyIndexMetadata(final NodeServicesProvider nodeServicesProvider, IndexMetaData metaData) throws IOException {
+        final List<Closeable> closeables = new ArrayList<>();
+        try {
+            IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(settings, new IndexFieldDataCache.Listener() {});
+            closeables.add(indicesFieldDataCache);
+            IndicesQueryCache indicesQueryCache = new IndicesQueryCache(settings);
+            closeables.add(indicesQueryCache);
+            // this will also fail if some plugin fails etc. which is nice since we can verify that early
+            final IndexService service = createIndexService("metadata verification", nodeServicesProvider,
+                metaData, indicesQueryCache, indicesFieldDataCache, Collections.emptyList());
+            for (ObjectCursor<MappingMetaData> typeMapping : metaData.getMappings().values()) {
+                // don't apply the default mapping, it has been applied when the mapping was created
+                service.mapperService().merge(typeMapping.value.type(), typeMapping.value.source(),
+                    MapperService.MergeReason.MAPPING_RECOVERY, true);
+            }
+            closeables.add(() -> service.close("metadata verification", false));
+        } finally {
+            IOUtils.close(closeables);
+        }
     }
 
     /**

core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java

@@ -19,9 +19,13 @@
 
 package org.elasticsearch.gateway;
 
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.client.Requests;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
@@ -31,7 +35,11 @@ import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.indices.IndexClosedException;
+import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
@@ -40,6 +48,7 @@ import org.elasticsearch.test.InternalTestCluster.RestartCallback;
 
 import static org.elasticsearch.common.settings.Settings.settingsBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
+import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.equalTo;
@@ -364,4 +373,123 @@ public class GatewayIndexStateIT extends ESIntegTestCase {
         logger.info("--> verify the doc is there");
         assertThat(client().prepareGet("test", "type1", "1").execute().actionGet().isExists(), equalTo(true));
     }
+
+    /**
+     * This test really tests worst case scenario where we have a broken setting or any setting that prevents an index from being
+     * allocated in our metadata that we recover. In that case we now have the ability to check the index on local recovery from disk
+     * if it is sane and if we can successfully create an IndexService. This also includes plugins etc.
+     */
+    public void testRecoverBrokenIndexMetadata() throws Exception {
+        logger.info("--> starting one node");
+        internalCluster().startNode();
+        logger.info("--> indexing a simple document");
+        client().prepareIndex("test", "type1", "1").setSource("field1", "value1").setRefresh(true).execute().actionGet();
+        logger.info("--> waiting for green status");
+        if (usually()) {
+            ensureYellow();
+        } else {
+            internalCluster().startNode();
+            client().admin().cluster()
+                .health(Requests.clusterHealthRequest()
+                    .waitForGreenStatus()
+                    .waitForEvents(Priority.LANGUID)
+                    .waitForRelocatingShards(0).waitForNodes("2")).actionGet();
+        }
+        ClusterState state = client().admin().cluster().prepareState().get().getState();
+        IndexMetaData metaData = state.getMetaData().index("test");
+        for (NodeEnvironment services : internalCluster().getInstances(NodeEnvironment.class)) {
+            IndexMetaData brokenMeta = IndexMetaData.builder(metaData).settings(Settings.builder().put(metaData.getSettings())
+                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_0_0_beta1.id)
+                 // this is invalid but should be archived
+                .put("index.similarity.BM25.type", "classic")
+                 // this one is not validated ahead of time and breaks allocation
+                .put("index.analysis.filter.myCollator.type", "icu_collation")
+            ).build();
+            IndexMetaData.FORMAT.write(brokenMeta, brokenMeta.getVersion(), services.indexPaths(brokenMeta.getIndex()));
+        }
+        internalCluster().fullRestart();
+        // ensureGreen(closedIndex) waits for the index to show up in the metadata
+        // this is crucial otherwise the state call below might not contain the index yet
+        ensureGreen(metaData.getIndex().getName());
+        state = client().admin().cluster().prepareState().get().getState();
+        assertEquals(IndexMetaData.State.CLOSE, state.getMetaData().index(metaData.getIndex()).getState());
+        assertEquals("classic", state.getMetaData().index(metaData.getIndex()).getSettings().get("archived.index.similarity.BM25.type"));
+        // try to open it with the broken setting - fail again!
+        ElasticsearchException ex = expectThrows(ElasticsearchException.class, () -> client().admin().indices().prepareOpen("test").get());
+        assertEquals(ex.getMessage(), "Failed to verify index " + metaData.getIndex());
+        assertNotNull(ex.getCause());
+        assertEquals(IllegalArgumentException.class, ex.getCause().getClass());
+        assertEquals(ex.getCause().getMessage(), "Unknown tokenfilter type [icu_collation] for [myCollator]");
+
+        client().admin().indices().prepareUpdateSettings()
+            .setSettings(Settings.builder().putNull("index.analysis.filter.myCollator.type")).get();
+        client().admin().indices().prepareOpen("test").get();
+        ensureYellow();
+        logger.info("--> verify 1 doc in the index");
+        assertHitCount(client().prepareSearch().setQuery(matchAllQuery()).get(), 1L);
+    }
+
+    /**
+     * This test really tests worst case scenario where we have a missing analyzer setting.
+     * In that case we now have the ability to check the index on local recovery from disk
+     * if it is sane and if we can successfully create an IndexService.
+     * This also includes plugins etc.
+     */
+    public void testRecoverMissingAnalyzer() throws Exception {
+        logger.info("--> starting one node");
+        internalCluster().startNode();
+        prepareCreate("test").setSettings(Settings.builder()
+            .put("index.analysis.analyzer.test.tokenizer", "keyword")
+            .put("index.number_of_shards", "1"))
+            .addMapping("type1", "{\n" +
+                "    \"type1\": {\n" +
+                "      \"properties\": {\n" +
+                "        \"field1\": {\n" +
+                "          \"type\": \"text\",\n" +
+                "          \"analyzer\": \"test\"\n" +
+                "        }\n" +
+                "      }\n" +
+                "    }\n" +
+                "  }}").get();
+        logger.info("--> indexing a simple document");
+        client().prepareIndex("test", "type1", "1").setSource("field1", "value one").setRefresh(true).execute().actionGet();
+        logger.info("--> waiting for green status");
+        if (usually()) {
+            ensureYellow();
+        } else {
+            internalCluster().startNode();
+            client().admin().cluster()
+                .health(Requests.clusterHealthRequest()
+                    .waitForGreenStatus()
+                    .waitForEvents(Priority.LANGUID)
+                    .waitForRelocatingShards(0).waitForNodes("2")).actionGet();
+        }
+        ClusterState state = client().admin().cluster().prepareState().get().getState();
+        IndexMetaData metaData = state.getMetaData().index("test");
+        for (NodeEnvironment services : internalCluster().getInstances(NodeEnvironment.class)) {
+            IndexMetaData brokenMeta = IndexMetaData.builder(metaData).settings(metaData.getSettings()
+                .filter((s) -> "index.analysis.analyzer.test.tokenizer".equals(s) == false)).build();
+            IndexMetaData.FORMAT.write(brokenMeta, brokenMeta.getVersion(), services.indexPaths(brokenMeta.getIndex()));
+        }
+        internalCluster().fullRestart();
+        // ensureGreen(closedIndex) waits for the index to show up in the metadata
+        // this is crucial otherwise the state call below might not contain the index yet
+        ensureGreen(metaData.getIndex().getName());
+        state = client().admin().cluster().prepareState().get().getState();
+        assertEquals(IndexMetaData.State.CLOSE, state.getMetaData().index(metaData.getIndex()).getState());
+
+        // try to open it with the broken setting - fail again!
+        ElasticsearchException ex = expectThrows(ElasticsearchException.class, () -> client().admin().indices().prepareOpen("test").get());
+        assertEquals(ex.getMessage(), "Failed to verify index " + metaData.getIndex());
+        assertNotNull(ex.getCause());
+        assertEquals(MapperParsingException.class, ex.getCause().getClass());
+        assertEquals(ex.getCause().getMessage(), "analyzer [test] not found for field [field1]");
+
+        client().admin().indices().prepareUpdateSettings()
+            .setSettings(Settings.builder().put("index.analysis.analyzer.test.tokenizer", "keyword")).get();
+        client().admin().indices().prepareOpen("test").get();
+        ensureYellow();
+        logger.info("--> verify 1 doc in the index");
+        assertHitCount(client().prepareSearch().setQuery(matchQuery("field1", "value one")).get(), 1L);
+    }
 }

core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java

@@ -40,8 +40,7 @@ public class GatewayServiceTests extends ESTestCase {
                 .put("http.enabled", "false")
                 .put("discovery.type", "local")
                 .put(settings.build()).build(),
-                null, clusterService, null, null, null, null, new NoopDiscovery());
-
+                null, clusterService, null, null, null, null, new NoopDiscovery(), null, null);
     }
 
     public void testDefaultRecoverAfterTime() throws IOException {