Allow shards to be allocated if leftover shard from different index exists.

If an index name is reused but a leftover shard still exists on any node
we fail repeatedly to allocate the shard since we now check the index UUID
before reusing data. This commit allows to recover even if there is such a
leftover shard by deleting the leftover shard.

Closes #10677

core/src/main/java/org/elasticsearch/index/IndexService.java

@@ -284,8 +284,20 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone
         boolean success = false;
         Injector shardInjector = null;
         try {
-
-            ShardPath path = ShardPath.loadShardPath(logger, nodeEnv, shardId, indexSettings);
+            lock = nodeEnv.shardLock(shardId, TimeUnit.SECONDS.toMillis(5));
+            ShardPath path;
+            try {
+                path = ShardPath.loadShardPath(logger, nodeEnv, shardId, indexSettings);
+            } catch (IllegalStateException ex) {
+                logger.warn("{} failed to load shard path, trying to archive leftover", shardId);
+                try {
+                    ShardPath.deleteLeftoverShardDirectory(logger, nodeEnv, lock, indexSettings);
+                    path = ShardPath.loadShardPath(logger, nodeEnv, shardId, indexSettings);
+                } catch (Throwable t) {
+                    t.addSuppressed(ex);
+                    throw t;
+                }
+            }
             if (path == null) {
                 path = ShardPath.selectNewPathForShard(nodeEnv, shardId, indexSettings, getAvgShardSizeInBytes(), this);
                 logger.debug("{} creating using a new path [{}]", shardId, path);
@@ -293,7 +305,6 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone
                 logger.debug("{} creating using an existing path [{}]", shardId, path);
             }
 
-            lock = nodeEnv.shardLock(shardId, TimeUnit.SECONDS.toMillis(5));
             if (shards.containsKey(shardId.id())) {
                 throw new IndexShardAlreadyExistsException(shardId + " already exists");
             }

core/src/main/java/org/elasticsearch/index/shard/ShardPath.java

@@ -18,10 +18,12 @@
  */
 package org.elasticsearch.index.shard;
 
+import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.env.ShardLock;
 import org.elasticsearch.index.settings.IndexSettings;
 
 import java.io.IOException;
@@ -88,7 +90,7 @@ public final class ShardPath {
         for (Path path : paths) {
             ShardStateMetaData load = ShardStateMetaData.FORMAT.loadLatestState(logger, path);
             if (load != null) {
-                if ((load.indexUUID.equals(indexUUID) || IndexMetaData.INDEX_UUID_NA_VALUE.equals(load.indexUUID)) == false) {
+                if (load.indexUUID.equals(indexUUID) == false && IndexMetaData.INDEX_UUID_NA_VALUE.equals(load.indexUUID) == false) {
                     logger.warn("{} found shard on path: [{}] with a different index UUID - this shard seems to be leftover from a different index with the same name. Remove the leftover shard in order to reuse the path with the current index", shardId, path);
                     throw new IllegalStateException(shardId + " index UUID in shard state was: " + load.indexUUID + " expected: " + indexUUID + " on shard path: " + path);
                 }
@@ -115,6 +117,26 @@ public final class ShardPath {
         }
     }
 
+    /**
+     * This method tries to delete left-over shards where the index name has been reused but the UUID is different
+     * to allow the new shard to be allocated.
+     */
+    public static void deleteLeftoverShardDirectory(ESLogger logger, NodeEnvironment env, ShardLock lock, @IndexSettings Settings indexSettings) throws IOException {
+        final String indexUUID = indexSettings.get(IndexMetaData.SETTING_INDEX_UUID, IndexMetaData.INDEX_UUID_NA_VALUE);
+        final Path[] paths = env.availableShardPaths(lock.getShardId());
+        for (Path path : paths) {
+            ShardStateMetaData load = ShardStateMetaData.FORMAT.loadLatestState(logger, path);
+            if (load != null) {
+                if (load.indexUUID.equals(indexUUID) == false && IndexMetaData.INDEX_UUID_NA_VALUE.equals(load.indexUUID) == false) {
+                    logger.warn("{} deleting leftover shard on path: [{}] with a different index UUID", lock.getShardId(), path);
+                    assert Files.isDirectory(path) : path + " is not a directory";
+                    NodeEnvironment.acquireFSLockForPaths(indexSettings, paths);
+                    IOUtils.rm(path);
+                }
+            }
+        }
+    }
+
     /** Maps each path.data path to a "guess" of how many bytes the shards allocated to that path might additionally use over their
      *  lifetime; we do this so a bunch of newly allocated shards won't just all go the path with the most free space at this moment. */
     private static Map<Path,Long> getEstimatedReservedBytes(NodeEnvironment env, long avgShardSizeInBytes, Iterable<IndexShard> shards) throws IOException {

core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java

@@ -23,11 +23,13 @@ import org.apache.lucene.store.LockObtainFailedException;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.stats.IndexStats;
+import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.NodeEnvironment;
@@ -46,7 +48,9 @@ import org.elasticsearch.test.VersionUtils;
 import org.junit.Test;
 
 import java.io.IOException;
+import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
 import java.util.HashSet;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
@@ -54,6 +58,7 @@ import java.util.concurrent.ExecutionException;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.*;
 import static org.elasticsearch.common.settings.Settings.settingsBuilder;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
 import static org.hamcrest.Matchers.equalTo;
 
@@ -406,4 +411,28 @@ public class IndexShardTests extends ElasticsearchSingleNodeTest {
         client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().put(IndexMetaData.SETTING_PRIORITY, 400).build()).get();
         assertEquals(400, indexSettingsService.getSettings().getAsInt(IndexMetaData.SETTING_PRIORITY, 0).intValue());
     }
+
+    public void testRecoverIntoLeftover() throws IOException {
+        createIndex("test");
+        ensureGreen("test");
+        client().prepareIndex("test", "bar", "1").setSource("{}").setRefresh(true).get();
+        client().admin().indices().prepareFlush("test").get();
+        SearchResponse response = client().prepareSearch("test").get();
+        assertHitCount(response, 1l);
+        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
+        IndexService test = indicesService.indexService("test");
+        IndexShard shard = test.shard(0);
+        ShardPath shardPath = shard.shardPath();
+        Path dataPath = shardPath.getDataPath();
+        client().admin().indices().prepareClose("test").get();
+        Path tempDir = createTempDir();
+        Files.move(dataPath, tempDir.resolve("test"));
+        client().admin().indices().prepareDelete("test").get();
+        Files.createDirectories(dataPath.getParent());
+        Files.move(tempDir.resolve("test"), dataPath);
+        createIndex("test");
+        ensureGreen("test");
+        response = client().prepareSearch("test").get();
+        assertHitCount(response, 0l);
+    }
 }