[STORE] Remove `memory`/ `ram` store

The RAM store is discuraged for production usage anyway and
we don't test it in our randomized infrastructure. This commit
removes it for `2.0`

docs/reference/index-modules/store.asciidoc

@@ -122,9 +122,3 @@ the impact on the operating system. All other files are opened using Lucene `NIO
 Address space settings (<<vm-max-map-count>>) might also apply if your term
 dictionaries are large.
 
-[float]
-[[store-memory]]
-=== Memory
-
-The `memory` type stores the index in main memory, using Lucene's
-`RamIndexStore`.

docs/reference/migration/migrate_2_0.asciidoc

@@ -48,3 +48,7 @@ In addition, the following node settings related to routing have been deprecated
   This was an undocumented setting that allowed to take the `_type` of the
   document into account when computing its shard (default: `false`). `false` is
   now enforced on new indices.
+
+=== Store
+
+The `memory` / `ram` store (`index.store.type`) option was removed in Elasticsearch 2.0.

src/main/java/org/elasticsearch/env/NodeEnvironment.java

@@ -548,5 +548,8 @@ public class NodeEnvironment extends AbstractComponent implements Closeable{
         return paths;
     }
 
+    Settings getSettings() { // for testing
+        return settings;
+    }
 
 }

src/main/java/org/elasticsearch/index/gateway/local/LocalIndexShardGateway.java

@@ -130,7 +130,7 @@ public class LocalIndexShardGateway extends AbstractIndexShardComponent implemen
                     } catch (Throwable e1) {
                         files += " (failure=" + ExceptionsHelper.detailedMessage(e1) + ")";
                     }
-                    if (indexShouldExists && indexShard.indexService().store().persistent()) {
+                    if (indexShouldExists) {
                         throw new IndexShardGatewayRecoveryException(shardId(), "shard allocated for local recovery (post api), should exist, but doesn't, current files: " + files, e);
                     }
                 }

src/main/java/org/elasticsearch/index/store/IndexStore.java

@@ -24,18 +24,15 @@ import org.elasticsearch.index.CloseableIndexComponent;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.store.IndicesStore;
 
+import java.io.File;
 import java.io.IOException;
+import java.nio.file.Path;
 
 /**
  * Index store is an index level information of the {@link Store} each shard will use.
  */
 public interface IndexStore extends CloseableIndexComponent {
 
-    /**
-     * Is the store a persistent store that can survive full restarts.
-     */
-    boolean persistent();
-
     IndicesStore indicesStore();
 
     /**
@@ -59,4 +56,6 @@ public interface IndexStore extends CloseableIndexComponent {
      * Deletes this shard store since its no longer allocated.
      */
     void deleteUnallocated(ShardId shardId) throws IOException;
+
+    Path[] shardIndexLocations(ShardId shardId);
 }

src/main/java/org/elasticsearch/index/store/IndexStoreModule.java

@@ -31,7 +31,6 @@ import org.elasticsearch.index.store.fs.DefaultFsIndexStoreModule;
 import org.elasticsearch.index.store.fs.MmapFsIndexStoreModule;
 import org.elasticsearch.index.store.fs.NioFsIndexStoreModule;
 import org.elasticsearch.index.store.fs.SimpleFsIndexStoreModule;
-import org.elasticsearch.index.store.ram.RamIndexStoreModule;
 
 /**
  *
@@ -57,26 +56,8 @@ public class IndexStoreModule extends AbstractModule implements SpawnModules {
                 return super.match(setting) || "simple_fs".equalsIgnoreCase(setting);
             }
         },
-        RAM {
-            public  boolean fsStore() {
-                return false;
-            }
-        },
-        MEMORY {
-            public  boolean fsStore() {
-                return false;
-            }
-        },
         FS,
         DEFAULT,;
-
-        /**
-         * Returns true iff this store type is a filesystem based store.
-         */
-        public  boolean fsStore() {
-            return true;
-        }
-
         /**
          * Returns true iff this settings matches the type.
          */
@@ -104,11 +85,7 @@ public class IndexStoreModule extends AbstractModule implements SpawnModules {
             indexStoreModule = SimpleFsIndexStoreModule.class;
         }
         String storeType = settings.get("index.store.type");
-        if (Type.RAM.name().equalsIgnoreCase(storeType)) {
-            indexStoreModule = RamIndexStoreModule.class;
-        } else if (Type.MEMORY.match(storeType)) {
-            indexStoreModule = RamIndexStoreModule.class;
-        } else if (Type.FS.match(storeType)) {
+        if (Type.FS.match(storeType)) {
             // nothing to set here ... (we default to fs)
         } else if (Type.SIMPLEFS.match(storeType)) {
             indexStoreModule = SimpleFsIndexStoreModule.class;

src/main/java/org/elasticsearch/index/store/fs/DefaultFsDirectoryService.java

@@ -29,6 +29,7 @@ import org.elasticsearch.index.store.IndexStore;
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.file.Path;
 import java.util.Collections;
 import java.util.Set;
 
@@ -48,7 +49,7 @@ public class DefaultFsDirectoryService extends FsDirectoryService {
     }
 
     @Override
-    protected Directory newFSDirectory(File location, LockFactory lockFactory) throws IOException {
-        return new FileSwitchDirectory(PRIMARY_EXTENSIONS, new MMapDirectory(location.toPath(), lockFactory), new NIOFSDirectory(location.toPath(), lockFactory), true);
+    protected Directory newFSDirectory(Path location, LockFactory lockFactory) throws IOException {
+        return new FileSwitchDirectory(PRIMARY_EXTENSIONS, new MMapDirectory(location, lockFactory), new NIOFSDirectory(location, lockFactory), true);
     }
 }

src/main/java/org/elasticsearch/index/store/fs/DefaultFsIndexStore.java

@@ -26,12 +26,13 @@ import org.elasticsearch.index.Index;
 import org.elasticsearch.index.service.IndexService;
 import org.elasticsearch.index.settings.IndexSettings;
 import org.elasticsearch.index.store.DirectoryService;
+import org.elasticsearch.index.store.support.AbstractIndexStore;
 import org.elasticsearch.indices.store.IndicesStore;
 
 /**
  *
  */
-public final class DefaultFsIndexStore extends FsIndexStore {
+public final class DefaultFsIndexStore extends AbstractIndexStore {
 
     @Inject
     public DefaultFsIndexStore(Index index, @IndexSettings Settings indexSettings, IndexService indexService, IndicesStore indicesStore, NodeEnvironment nodeEnv) {

src/main/java/org/elasticsearch/index/store/fs/FsDirectoryService.java

@@ -30,18 +30,20 @@ import org.elasticsearch.index.store.IndexStore;
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
 
 /**
  */
 public abstract class FsDirectoryService extends DirectoryService implements StoreRateLimiting.Listener, StoreRateLimiting.Provider {
 
-    protected final FsIndexStore indexStore;
+    protected final IndexStore indexStore;
 
     private final CounterMetric rateLimitingTimeInNanos = new CounterMetric();
 
     public FsDirectoryService(ShardId shardId, @IndexSettings Settings indexSettings, IndexStore indexStore) {
         super(shardId, indexSettings);
-        this.indexStore = (FsIndexStore) indexStore;
+        this.indexStore = indexStore;
     }
 
     @Override
@@ -69,17 +71,17 @@ public abstract class FsDirectoryService extends DirectoryService implements Sto
     
     @Override
     public Directory[] build() throws IOException {
-        File[] locations = indexStore.shardIndexLocations(shardId);
+        Path[] locations = indexStore.shardIndexLocations(shardId);
         Directory[] dirs = new Directory[locations.length];
         for (int i = 0; i < dirs.length; i++) {
-            FileSystemUtils.mkdirs(locations[i]);
+            Files.createDirectories(locations[i]);
             Directory wrapped = newFSDirectory(locations[i], buildLockFactory());
             dirs[i] = new RateLimitedFSDirectory(wrapped, this, this) ;
         }
         return dirs;
     }
     
-    protected abstract Directory newFSDirectory(File location, LockFactory lockFactory) throws IOException;
+    protected abstract Directory newFSDirectory(Path location, LockFactory lockFactory) throws IOException;
 
     @Override
     public void onPause(long nanos) {

src/main/java/org/elasticsearch/index/store/fs/FsIndexStore.java

@@ -1,113 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.store.fs;
-
-import org.elasticsearch.ElasticsearchIllegalStateException;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.NodeEnvironment;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.index.service.IndexService;
-import org.elasticsearch.index.settings.IndexSettings;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.index.store.support.AbstractIndexStore;
-import org.elasticsearch.indices.store.IndicesStore;
-
-import java.io.File;
-import java.io.IOException;
-
-/**
- *
- */
-public abstract class FsIndexStore extends AbstractIndexStore {
-
-    private final NodeEnvironment nodeEnv;
-
-    private final File[] locations;
-
-    public FsIndexStore(Index index, @IndexSettings Settings indexSettings, IndexService indexService, IndicesStore indicesStore, NodeEnvironment nodeEnv) {
-        super(index, indexSettings, indexService, indicesStore);
-        this.nodeEnv = nodeEnv;
-        if (nodeEnv.hasNodeFile()) {
-            this.locations = nodeEnv.indexLocations(index);
-        } else {
-            this.locations = null;
-        }
-    }
-
-    @Override
-    public boolean persistent() {
-        return true;
-    }
-
-    @Override
-    public boolean canDeleteUnallocated(ShardId shardId) {
-        if (locations == null) {
-            return false;
-        }
-        if (indexService.hasShard(shardId.id())) {
-            return false;
-        }
-        for (File location : shardLocations(shardId)) {
-            if (location.exists()) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    @Override
-    public void deleteUnallocated(ShardId shardId) throws IOException {
-        if (locations == null) {
-            return;
-        }
-        if (indexService.hasShard(shardId.id())) {
-            throw new ElasticsearchIllegalStateException(shardId + " allocated, can't be deleted");
-        }
-        try {
-            nodeEnv.deleteShardDirectorySafe(shardId);
-        } catch (Exception ex) {
-            logger.debug("failed to delete shard locations", ex);
-        }
-    }
-
-    public File[] shardLocations(ShardId shardId) {
-        return nodeEnv.shardLocations(shardId);
-    }
-
-    public File[] shardIndexLocations(ShardId shardId) {
-        File[] shardLocations = shardLocations(shardId);
-        File[] shardIndexLocations = new File[shardLocations.length];
-        for (int i = 0; i < shardLocations.length; i++) {
-            shardIndexLocations[i] = new File(shardLocations[i], "index");
-        }
-        return shardIndexLocations;
-    }
-
-    // not used currently, but here to state that this store also defined a file based translog location
-
-    public File[] shardTranslogLocations(ShardId shardId) {
-        File[] shardLocations = shardLocations(shardId);
-        File[] shardTranslogLocations = new File[shardLocations.length];
-        for (int i = 0; i < shardLocations.length; i++) {
-            shardTranslogLocations[i] = new File(shardLocations[i], "translog");
-        }
-        return shardTranslogLocations;
-    }
-}

src/main/java/org/elasticsearch/index/store/fs/MmapFsDirectoryService.java

@@ -30,6 +30,7 @@ import org.elasticsearch.index.store.IndexStore;
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.file.Path;
 
 /**
  */
@@ -41,7 +42,7 @@ public class MmapFsDirectoryService extends FsDirectoryService {
     }
 
     @Override
-    protected Directory newFSDirectory(File location, LockFactory lockFactory) throws IOException {
-        return new MMapDirectory(location.toPath(), buildLockFactory());
+    protected Directory newFSDirectory(Path location, LockFactory lockFactory) throws IOException {
+        return new MMapDirectory(location, buildLockFactory());
     }
 }

src/main/java/org/elasticsearch/index/store/fs/MmapFsIndexStore.java

@@ -26,12 +26,13 @@ import org.elasticsearch.index.Index;
 import org.elasticsearch.index.service.IndexService;
 import org.elasticsearch.index.settings.IndexSettings;
 import org.elasticsearch.index.store.DirectoryService;
+import org.elasticsearch.index.store.support.AbstractIndexStore;
 import org.elasticsearch.indices.store.IndicesStore;
 
 /**
  *
  */
-public final class MmapFsIndexStore extends FsIndexStore {
+public final class MmapFsIndexStore extends AbstractIndexStore {
 
     @Inject
     public MmapFsIndexStore(Index index, @IndexSettings Settings indexSettings, IndexService indexService, IndicesStore indicesStore, NodeEnvironment nodeEnv) {

src/main/java/org/elasticsearch/index/store/fs/NioFsDirectoryService.java

@@ -30,6 +30,7 @@ import org.elasticsearch.index.store.IndexStore;
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.file.Path;
 
 /**
  */
@@ -41,7 +42,7 @@ public class NioFsDirectoryService extends FsDirectoryService {
     }
 
     @Override
-    protected Directory newFSDirectory(File location, LockFactory lockFactory) throws IOException {
-        return new NIOFSDirectory(location.toPath(), lockFactory);
+    protected Directory newFSDirectory(Path location, LockFactory lockFactory) throws IOException {
+        return new NIOFSDirectory(location, lockFactory);
     }
 }

src/main/java/org/elasticsearch/index/store/fs/NioFsIndexStore.java

@@ -26,12 +26,13 @@ import org.elasticsearch.index.Index;
 import org.elasticsearch.index.service.IndexService;
 import org.elasticsearch.index.settings.IndexSettings;
 import org.elasticsearch.index.store.DirectoryService;
+import org.elasticsearch.index.store.support.AbstractIndexStore;
 import org.elasticsearch.indices.store.IndicesStore;
 
 /**
  *
  */
-public final class NioFsIndexStore extends FsIndexStore {
+public final class NioFsIndexStore extends AbstractIndexStore {
 
     @Inject
     public NioFsIndexStore(Index index, @IndexSettings Settings indexSettings, IndexService indexService, IndicesStore indicesStore, NodeEnvironment nodeEnv) {

src/main/java/org/elasticsearch/index/store/fs/SimpleFsDirectoryService.java

@@ -30,6 +30,7 @@ import org.elasticsearch.index.store.IndexStore;
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.file.Path;
 
 /**
  */
@@ -41,7 +42,7 @@ public class SimpleFsDirectoryService extends FsDirectoryService {
     }
 
     @Override
-    protected Directory newFSDirectory(File location, LockFactory lockFactory) throws IOException {
-        return new SimpleFSDirectory(location.toPath(), lockFactory);
+    protected Directory newFSDirectory(Path location, LockFactory lockFactory) throws IOException {
+        return new SimpleFSDirectory(location, lockFactory);
     }
 }

src/main/java/org/elasticsearch/index/store/fs/SimpleFsIndexStore.java

@@ -26,12 +26,13 @@ import org.elasticsearch.index.Index;
 import org.elasticsearch.index.service.IndexService;
 import org.elasticsearch.index.settings.IndexSettings;
 import org.elasticsearch.index.store.DirectoryService;
+import org.elasticsearch.index.store.support.AbstractIndexStore;
 import org.elasticsearch.indices.store.IndicesStore;
 
 /**
  *
  */
-public final class SimpleFsIndexStore extends FsIndexStore {
+public final class SimpleFsIndexStore extends AbstractIndexStore {
 
     @Inject
     public SimpleFsIndexStore(Index index, @IndexSettings Settings indexSettings, IndexService indexService, IndicesStore indicesStore, NodeEnvironment nodeEnv) {

src/main/java/org/elasticsearch/index/store/ram/RamDirectoryService.java

@@ -1,72 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.store.ram;
-
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.RAMDirectory;
-import org.apache.lucene.store.RAMFile;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.settings.IndexSettings;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.index.store.DirectoryService;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-
-/**
- */
-public final class RamDirectoryService extends DirectoryService {
-
-    @Inject
-    public RamDirectoryService(ShardId shardId, @IndexSettings Settings indexSettings) {
-        super(shardId, indexSettings);
-    }
-
-    @Override
-    public long throttleTimeInNanos() {
-        return 0;
-    }
-
-    @Override
-    public Directory[] build() {
-        return new Directory[]{new CustomRAMDirectory()};
-    }
-
-    static class CustomRAMDirectory extends RAMDirectory {
-
-        public synchronized void renameTo(String from, String to) throws IOException {
-            RAMFile fromFile = fileMap.get(from);
-            if (fromFile == null)
-                throw new FileNotFoundException(from);
-            RAMFile toFile = fileMap.get(to);
-            if (toFile != null) {
-                sizeInBytes.addAndGet(-fileLength(to));
-            }
-            fileMap.put(to, fromFile);
-            fileMap.remove(from);
-        }
-
-        @Override
-        public String toString() {
-            return "ram";
-        }
-    }
-}

src/main/java/org/elasticsearch/index/store/ram/RamIndexStore.java

@@ -1,50 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.store.ram;
-
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.index.service.IndexService;
-import org.elasticsearch.index.settings.IndexSettings;
-import org.elasticsearch.index.store.DirectoryService;
-import org.elasticsearch.index.store.support.AbstractIndexStore;
-import org.elasticsearch.indices.store.IndicesStore;
-
-/**
- *
- */
-public class RamIndexStore extends AbstractIndexStore {
-
-    @Inject
-    public RamIndexStore(Index index, @IndexSettings Settings indexSettings, IndexService indexService, IndicesStore indicesStore) {
-        super(index, indexSettings, indexService, indicesStore);
-    }
-
-    @Override
-    public boolean persistent() {
-        return false;
-    }
-
-    @Override
-    public Class<? extends DirectoryService> shardDirectory() {
-        return RamDirectoryService.class;
-    }
-}

src/main/java/org/elasticsearch/index/store/ram/RamIndexStoreModule.java

@@ -1,34 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.store.ram;
-
-import org.elasticsearch.common.inject.AbstractModule;
-import org.elasticsearch.index.store.IndexStore;
-
-/**
- *
- */
-public class RamIndexStoreModule extends AbstractModule {
-
-    @Override
-    protected void configure() {
-        bind(IndexStore.class).to(RamIndexStore.class).asEagerSingleton();
-    }
-}

src/main/java/org/elasticsearch/index/store/support/AbstractIndexStore.java

@@ -21,8 +21,10 @@ package org.elasticsearch.index.store.support;
 
 import org.apache.lucene.store.StoreRateLimiting;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchIllegalStateException;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.AbstractIndexComponent;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.service.IndexService;
@@ -32,7 +34,10 @@ import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.store.IndexStore;
 import org.elasticsearch.indices.store.IndicesStore;
 
+import java.io.File;
 import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
 
 /**
  *
@@ -67,7 +72,9 @@ public abstract class AbstractIndexStore extends AbstractIndexComponent implemen
             }
         }
     }
+    private final NodeEnvironment nodeEnv;
 
+    private final File[] locations;
 
     protected final IndexService indexService;
 
@@ -81,7 +88,7 @@ public abstract class AbstractIndexStore extends AbstractIndexComponent implemen
 
     private final ApplySettings applySettings = new ApplySettings();
 
-    protected AbstractIndexStore(Index index, @IndexSettings Settings indexSettings, IndexService indexService, IndicesStore indicesStore) {
+    protected AbstractIndexStore(Index index, @IndexSettings Settings indexSettings, IndexService indexService, IndicesStore indicesStore, NodeEnvironment nodeEnv) {
         super(index, indexSettings);
         this.indexService = indexService;
         this.indicesStore = indicesStore;
@@ -99,6 +106,12 @@ public abstract class AbstractIndexStore extends AbstractIndexComponent implemen
         logger.debug("using index.store.throttle.type [{}], with index.store.throttle.max_bytes_per_sec [{}]", rateLimitingType, rateLimitingThrottle);
 
         indexService.settingsService().addListener(applySettings);
+        this.nodeEnv = nodeEnv;
+        if (nodeEnv.hasNodeFile()) {
+            this.locations = nodeEnv.indexLocations(index);
+        } else {
+            this.locations = null;
+        }
     }
 
     @Override
@@ -107,22 +120,53 @@ public abstract class AbstractIndexStore extends AbstractIndexComponent implemen
     }
 
     @Override
-    public boolean canDeleteUnallocated(ShardId shardId) {
-        return false;
+    public IndicesStore indicesStore() {
+        return indicesStore;
     }
 
     @Override
-    public void deleteUnallocated(ShardId shardId) throws IOException {
-        // do nothing here...
+    public StoreRateLimiting rateLimiting() {
+        return nodeRateLimiting ? indicesStore.rateLimiting() : this.rateLimiting;
     }
 
+
     @Override
-    public IndicesStore indicesStore() {
-        return indicesStore;
+    public boolean canDeleteUnallocated(ShardId shardId) {
+        if (locations == null) {
+            return false;
+        }
+        if (indexService.hasShard(shardId.id())) {
+            return false;
+        }
+        for (Path location : nodeEnv.shardPaths(shardId)) {
+            if (Files.exists(location)) {
+                return true;
+            }
+        }
+        return false;
     }
 
     @Override
-    public StoreRateLimiting rateLimiting() {
-        return nodeRateLimiting ? indicesStore.rateLimiting() : this.rateLimiting;
+    public void deleteUnallocated(ShardId shardId) throws IOException {
+        if (locations == null) {
+            return;
+        }
+        if (indexService.hasShard(shardId.id())) {
+            throw new ElasticsearchIllegalStateException(shardId + " allocated, can't be deleted");
+        }
+        try {
+            nodeEnv.deleteShardDirectorySafe(shardId);
+        } catch (Exception ex) {
+            logger.debug("failed to delete shard locations", ex);
+        }
+    }
+
+    public Path[] shardIndexLocations(ShardId shardId) {
+        Path[] shardLocations = nodeEnv.shardPaths(shardId);
+        Path[] shardIndexLocations = new Path[shardLocations.length];
+        for (int i = 0; i < shardLocations.length; i++) {
+            shardIndexLocations[i] = shardLocations[i].resolve("index");
+        }
+        return shardIndexLocations;
     }
 }

src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java

@@ -42,11 +42,10 @@ public class NodeEnvironmentTests extends ElasticsearchTestCase {
 
     @Test
     public void testNodeLockSingleEnvironment() throws IOException {
-        String[] dataPaths = tmpPaths();
-        Settings settings = ImmutableSettings.builder()
-                .put(nodeEnvSettings(dataPaths))
-                .put("node.max_local_storage_nodes", 1).build();
-        NodeEnvironment env = new NodeEnvironment(settings, new Environment(settings));
+        NodeEnvironment env = newNodeEnvironment(ImmutableSettings.builder()
+                .put("node.max_local_storage_nodes", 1).build());
+        Settings settings = env.getSettings();
+        String[] dataPaths = env.getSettings().getAsArray("path.data");
 
         try {
             new NodeEnvironment(settings, new Environment(settings));
@@ -70,10 +69,9 @@ public class NodeEnvironmentTests extends ElasticsearchTestCase {
 
     @Test
     public void testNodeLockMultipleEnvironment() throws IOException {
-        String[] dataPaths = tmpPaths();
-        Settings settings = nodeEnvSettings(dataPaths);
-        NodeEnvironment first = new NodeEnvironment(settings, new Environment(settings));
-        NodeEnvironment second = new NodeEnvironment(settings, new Environment(settings));
+        final NodeEnvironment first = newNodeEnvironment();
+        String[] dataPaths = first.getSettings().getAsArray("path.data");
+        NodeEnvironment second = new NodeEnvironment(first.getSettings(), new Environment(first.getSettings()));
         assertEquals(first.nodeDataPaths().length, dataPaths.length);
         assertEquals(second.nodeDataPaths().length, dataPaths.length);
         for (int i = 0; i < dataPaths.length; i++) {
@@ -84,8 +82,7 @@ public class NodeEnvironmentTests extends ElasticsearchTestCase {
 
     @Test
     public void testShardLock() throws IOException {
-        Settings settings = nodeEnvSettings(tmpPaths());
-        NodeEnvironment env = new NodeEnvironment(settings, new Environment(settings));
+        final NodeEnvironment env = newNodeEnvironment();
 
         ShardLock fooLock = env.shardLock(new ShardId("foo", 1));
         assertEquals(new ShardId("foo", 1), fooLock.getShardId());
@@ -126,8 +123,7 @@ public class NodeEnvironmentTests extends ElasticsearchTestCase {
 
     @Test
     public void testGetAllIndices() throws Exception {
-        Settings settings = nodeEnvSettings(tmpPaths());
-        NodeEnvironment env = new NodeEnvironment(settings, new Environment(settings));
+        final NodeEnvironment env = newNodeEnvironment();
         final int numIndices = randomIntBetween(1, 10);
         for (int i = 0; i < numIndices; i++) {
             for (Path path : env.indexPaths(new Index("foo" + i))) {
@@ -145,9 +141,7 @@ public class NodeEnvironmentTests extends ElasticsearchTestCase {
 
     @Test
     public void testDeleteSafe() throws IOException {
-        Settings settings = nodeEnvSettings(tmpPaths());
-        NodeEnvironment env = new NodeEnvironment(settings, new Environment(settings));
-
+        final NodeEnvironment env = newNodeEnvironment();
         ShardLock fooLock = env.shardLock(new ShardId("foo", 1));
         assertEquals(new ShardId("foo", 1), fooLock.getShardId());
 
@@ -200,8 +194,7 @@ public class NodeEnvironmentTests extends ElasticsearchTestCase {
 
     @Test
     public void testGetAllShards() throws Exception {
-        Settings settings = nodeEnvSettings(tmpPaths());
-        NodeEnvironment env = new NodeEnvironment(settings, new Environment(settings));
+        final NodeEnvironment env = newNodeEnvironment();
         final int numIndices = randomIntBetween(1, 10);
         final Set<ShardId> createdShards = new HashSet<>();
         for (int i = 0; i < numIndices; i++) {
@@ -234,8 +227,7 @@ public class NodeEnvironmentTests extends ElasticsearchTestCase {
         class Int {
             int value = 0;
         }
-        Settings settings = nodeEnvSettings(tmpPaths());
-        final NodeEnvironment env = new NodeEnvironment(settings, new Environment(settings));
+        final NodeEnvironment env = newNodeEnvironment();
         final int shards = randomIntBetween(2, 10);
         final Int[] counts = new Int[shards];
         final AtomicInteger[] countsAtomic = new AtomicInteger[shards];
@@ -301,9 +293,15 @@ public class NodeEnvironmentTests extends ElasticsearchTestCase {
         return absPaths;
     }
 
-    private Settings nodeEnvSettings(String[] dataPaths) {
-        return ImmutableSettings.builder()
+    public NodeEnvironment newNodeEnvironment() throws IOException {
+        return newNodeEnvironment(ImmutableSettings.EMPTY);
+    }
+
+    public NodeEnvironment newNodeEnvironment(Settings settings) throws IOException {
+        Settings build = ImmutableSettings.builder()
+                .put(settings)
                 .put("path.home", newTempDir().getAbsolutePath())
-                .putArray("path.data", dataPaths).build();
+                .putArray("path.data", tmpPaths()).build();
+        return new NodeEnvironment(build, new Environment(build));
     }
 }

src/test/java/org/elasticsearch/index/engine/internal/InternalEngineTests.java

@@ -32,7 +32,8 @@ import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.IndexDeletionPolicy;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.store.Lock;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.RAMDirectory;
 import org.apache.lucene.util.LuceneTestCase.Slow;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
@@ -43,7 +44,6 @@ import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.ShardLock;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.analysis.AnalysisService;
@@ -69,7 +69,8 @@ import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.index.store.DirectoryService;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.index.store.distributor.LeastUsedDistributor;
-import org.elasticsearch.index.store.ram.RamDirectoryService;
+import org.elasticsearch.index.store.fs.SimpleFsDirectoryService;
+import org.elasticsearch.index.store.fs.SimpleFsIndexStore;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.index.translog.TranslogSizeMatcher;
 import org.elasticsearch.index.translog.fs.FsTranslog;
@@ -173,12 +174,33 @@ public class InternalEngineTests extends ElasticsearchTestCase {
     }
 
     protected Store createStore() throws IOException {
-        DirectoryService directoryService = new RamDirectoryService(shardId, EMPTY_SETTINGS);
+        final DirectoryService directoryService = new DirectoryService(shardId, EMPTY_SETTINGS) {
+            @Override
+            public Directory[] build() throws IOException {
+                return new Directory[] {new RAMDirectory() } ;
+            }
+
+            @Override
+            public long throttleTimeInNanos() {
+                return 0;
+            }
+        };
         return new Store(shardId, EMPTY_SETTINGS, null, directoryService, new LeastUsedDistributor(directoryService), new DummyShardLock(shardId));
     }
 
     protected Store createStoreReplica() throws IOException {
-        DirectoryService directoryService = new RamDirectoryService(shardId, EMPTY_SETTINGS);
+
+        final DirectoryService directoryService = new DirectoryService(shardId, EMPTY_SETTINGS) {
+            @Override
+            public Directory[] build() throws IOException {
+                return new Directory[] {new RAMDirectory() } ;
+            }
+
+            @Override
+            public long throttleTimeInNanos() {
+                return 0;
+            }
+        };
         return new Store(shardId, EMPTY_SETTINGS, null, directoryService, new LeastUsedDistributor(directoryService), new DummyShardLock(shardId));
     }
 

src/test/java/org/elasticsearch/index/mapper/FileBasedMappingsTests.java

@@ -82,7 +82,6 @@ public class FileBasedMappingsTests extends ElasticsearchTestCase {
                     .put("node.name", NAME)
                     .put("path.conf", configDir.getAbsolutePath())
                     .put("http.enabled", false)
-                    .put("index.store.type", "ram")
                     .put("gateway.type", "none")
                     .build();
 

src/test/java/org/elasticsearch/index/merge/policy/MergePolicySettingsTest.java

@@ -18,18 +18,17 @@
  */
 package org.elasticsearch.index.merge.policy;
 
-import org.apache.lucene.store.Lock;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.RAMDirectory;
 import org.elasticsearch.ElasticsearchIllegalArgumentException;
 import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.ShardLock;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.settings.IndexSettingsService;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.store.DirectoryService;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.index.store.distributor.LeastUsedDistributor;
-import org.elasticsearch.index.store.ram.RamDirectoryService;
 import org.elasticsearch.test.DummyShardLock;
 import org.elasticsearch.test.ElasticsearchTestCase;
 import org.junit.Test;
@@ -174,7 +173,17 @@ public class MergePolicySettingsTest extends ElasticsearchTestCase {
     }
 
     protected Store createStore(Settings settings) throws IOException {
-        DirectoryService directoryService = new RamDirectoryService(shardId, EMPTY_SETTINGS);
+        final DirectoryService directoryService = new DirectoryService(shardId, EMPTY_SETTINGS) {
+            @Override
+            public Directory[] build() throws IOException {
+                return new Directory[] { new RAMDirectory() } ;
+            }
+
+            @Override
+            public long throttleTimeInNanos() {
+                return 0;
+            }
+        };
         return new Store(shardId, settings, null, directoryService, new LeastUsedDistributor(directoryService), new DummyShardLock(shardId));
     }
 

src/test/java/org/elasticsearch/indices/memory/IndexingMemoryControllerTests.java

@@ -145,7 +145,6 @@ public class IndexingMemoryControllerTests extends ElasticsearchIntegrationTest
                         .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
                         .put(EsExecutors.PROCESSORS, 1) // limit the number of threads created
                         .put("http.enabled", false)
-                        .put("index.store.type", "ram")
                         .put("config.ignore_system_properties", true) // make sure we get what we set :)
                         .put("gateway.type", "none")
                         .put("indices.memory.interval", "100ms")

src/test/java/org/elasticsearch/indices/store/SimpleDistributorTests.java

@@ -43,9 +43,7 @@ public class SimpleDistributorTests extends ElasticsearchIntegrationTest {
     @Test
     public void testAvailableSpaceDetection() {
         for (IndexStoreModule.Type store : IndexStoreModule.Type.values()) {
-            if (store.fsStore()) {
-                createIndexWithStoreType("test", store, StrictDistributor.class.getCanonicalName());
-            }
+            createIndexWithStoreType("test", store, StrictDistributor.class.getCanonicalName());
         }
     }
 
@@ -104,12 +102,6 @@ public class SimpleDistributorTests extends ElasticsearchIntegrationTest {
         }
         assertThat(storeString, endsWith(", type=MERGE, rate=20.0)])"));
 
-        createIndexWithStoreType("test", IndexStoreModule.Type.MEMORY, "least_used");
-        storeString = getStoreDirectory("test", 0).toString();
-        logger.info(storeString);
-        dataPaths = dataPaths();
-        assertThat(storeString, equalTo("store(least_used[ram])"));
-
         createIndexWithoutRateLimitingStoreType("test", IndexStoreModule.Type.NIOFS, "least_used");
         storeString = getStoreDirectory("test", 0).toString();
         logger.info(storeString);

src/test/java/org/elasticsearch/test/ElasticsearchSingleNodeTest.java

@@ -101,7 +101,6 @@ public abstract class ElasticsearchSingleNodeTest extends ElasticsearchTestCase
                 .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
                 .put(EsExecutors.PROCESSORS, 1) // limit the number of threads created
                 .put("http.enabled", false)
-                .put("index.store.type", "ram")
                 .put("config.ignore_system_properties", true) // make sure we get what we set :)
                 .put("gateway.type", "none")).build();
         build.start();

src/test/java/org/elasticsearch/test/ElasticsearchTestCase.java

@@ -24,6 +24,7 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope;
 import com.google.common.base.Predicate;
 import com.google.common.collect.ImmutableList;
 import org.apache.lucene.store.MockDirectoryWrapper;
+import org.apache.lucene.store.SimpleFSDirectory;
 import org.apache.lucene.util.AbstractRandomizedTest;
 import org.apache.lucene.util.TimeUnits;
 import org.apache.lucene.uninverting.UninvertingReader;
@@ -35,9 +36,16 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsAbortPolicy;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.index.store.DirectoryService;
+import org.elasticsearch.index.store.fs.SimpleFsDirectoryService;
+import org.elasticsearch.index.store.fs.SimpleFsIndexStore;
 import org.elasticsearch.test.cache.recycler.MockBigArrays;
 import org.elasticsearch.test.cache.recycler.MockPageCacheRecycler;
 import org.elasticsearch.test.junit.listeners.LoggingListener;
@@ -536,4 +544,5 @@ public abstract class ElasticsearchTestCase extends AbstractRandomizedTest {
     public static boolean terminate(ThreadPool service) throws InterruptedException {
         return ThreadPool.terminate(service, 10, TimeUnit.SECONDS);
     }
+
 }

src/test/java/org/elasticsearch/test/InternalTestCluster.java

@@ -354,8 +354,6 @@ public final class InternalTestCluster extends TestCluster {
     private static Settings getRandomNodeSettings(long seed) {
         Random random = new Random(seed);
         Builder builder = ImmutableSettings.settingsBuilder()
-        /* use RAM directories in 10% of the runs */
-                //.put("index.store.type", random.nextInt(10) == 0 ? MockRamIndexStoreModule.class.getName() : MockFSIndexStoreModule.class.getName())
                 // decrease the routing schedule so new nodes will be added quickly - some random value between 30 and 80 ms
                 .put("cluster.routing.schedule", (30 + random.nextInt(50)) + "ms")
                         // default to non gateway

src/test/java/org/elasticsearch/test/store/MockDirectoryHelper.java

@@ -32,7 +32,6 @@ import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.store.DirectoryService;
 import org.elasticsearch.index.store.IndexStore;
 import org.elasticsearch.index.store.fs.*;
-import org.elasticsearch.index.store.ram.RamDirectoryService;
 
 import java.io.IOException;
 import java.util.Random;
@@ -117,10 +116,6 @@ public class MockDirectoryHelper {
         }
     }
 
-    public DirectoryService randomRamDirectoryService() {
-        return new RamDirectoryService(shardId, indexSettings);
-    }
-
     public static final class ElasticsearchMockDirectoryWrapper extends MockDirectoryWrapper {
 
         private final ESLogger logger;

src/test/java/org/elasticsearch/test/store/MockFSDirectoryService.java

@@ -49,6 +49,7 @@ import org.elasticsearch.test.ElasticsearchIntegrationTest;
 import java.io.File;
 import java.io.IOException;
 import java.io.PrintStream;
+import java.nio.file.Path;
 import java.util.EnumSet;
 import java.util.Random;
 
@@ -115,7 +116,7 @@ public class MockFSDirectoryService extends FsDirectoryService {
     }
     
     @Override
-    protected synchronized Directory newFSDirectory(File location, LockFactory lockFactory) throws IOException {
+    protected synchronized Directory newFSDirectory(Path location, LockFactory lockFactory) throws IOException {
         throw new UnsupportedOperationException();
     }
 

src/test/java/org/elasticsearch/test/store/MockFSIndexStore.java

@@ -25,10 +25,10 @@ import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.service.IndexService;
 import org.elasticsearch.index.store.DirectoryService;
-import org.elasticsearch.index.store.fs.FsIndexStore;
+import org.elasticsearch.index.store.support.AbstractIndexStore;
 import org.elasticsearch.indices.store.IndicesStore;
 
-public class MockFSIndexStore extends FsIndexStore {
+public class MockFSIndexStore extends AbstractIndexStore {
 
     @Inject
     public MockFSIndexStore(Index index, Settings indexSettings, IndexService indexService, IndicesStore indicesStore, NodeEnvironment nodeEnv) {

src/test/java/org/elasticsearch/test/store/MockRamDirectoryService.java

@@ -1,55 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.test.store;
-
-import org.apache.lucene.store.Directory;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.index.store.DirectoryService;
-import org.elasticsearch.test.ElasticsearchIntegrationTest;
-
-import java.io.IOException;
-import java.util.Random;
-
-public class MockRamDirectoryService extends DirectoryService {
-
-    private final MockDirectoryHelper helper;
-    private final DirectoryService delegateService;
-
-    @Inject
-    public MockRamDirectoryService(ShardId shardId, Settings indexSettings) {
-        super(shardId, indexSettings);
-        final long seed = indexSettings.getAsLong(ElasticsearchIntegrationTest.SETTING_INDEX_SEED, 0l);
-        Random random = new Random(seed);
-        helper = new MockDirectoryHelper(shardId, indexSettings, logger, random, seed);
-        delegateService = helper.randomRamDirectoryService();
-    }
-
-    @Override
-    public Directory[] build() throws IOException {
-        return helper.wrapAllInplace(delegateService.build());
-    }
-
-    @Override
-    public long throttleTimeInNanos() {
-        return delegateService.throttleTimeInNanos();
-    }
-}

src/test/java/org/elasticsearch/test/store/MockRamIndexStore.java

@@ -1,46 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.test.store;
-
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.index.service.IndexService;
-import org.elasticsearch.index.store.DirectoryService;
-import org.elasticsearch.index.store.support.AbstractIndexStore;
-import org.elasticsearch.indices.store.IndicesStore;
-
-public class MockRamIndexStore extends AbstractIndexStore{
-
-    @Inject
-    public MockRamIndexStore(Index index, Settings indexSettings, IndexService indexService, IndicesStore indicesStore) {
-        super(index, indexSettings, indexService, indicesStore);
-    }
-
-    @Override
-    public boolean persistent() {
-        return false;
-    }
-
-    @Override
-    public Class<? extends DirectoryService> shardDirectory() {
-        return MockRamDirectoryService.class;
-    }
-}

src/test/java/org/elasticsearch/test/store/MockRamIndexStoreModule.java

@@ -1,32 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.test.store;
-
-import org.elasticsearch.common.inject.AbstractModule;
-import org.elasticsearch.index.store.IndexStore;
-
-public class MockRamIndexStoreModule extends AbstractModule {
-
-    @Override
-    protected void configure() {
-        bind(IndexStore.class).to(MockRamIndexStore.class).asEagerSingleton();
-    }
-
-}