Randomize exceptions when file is deleted while still open.

This somehow emulates a behavior on windows but we should test the
other code paths as well.

src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java

@@ -21,6 +21,7 @@ package org.elasticsearch.indices.recovery;
 
 import com.google.common.collect.Sets;
 import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.ExceptionsHelper;
@@ -533,21 +534,18 @@ public class RecoveryTarget extends AbstractComponent {
             Exception failureToRename = null;
             if (!filesToRename.isEmpty()) {
                 // first, go and delete the existing ones
-                for (String fileToRename : filesToRename) {
-                    store.directory().deleteFile(fileToRename);
-                }
-                for (String fileToRename : filesToRename) {
-                    // now, rename the files...
+                final Directory directory = store.directory();
+                for (String file : filesToRename) {
                     try {
-                        store.renameFile(prefix + fileToRename, fileToRename);
-                    } catch (Exception e) {
-                        failureToRename = e;
-                        break;
+                        directory.deleteFile(file);
+                    } catch (Throwable ex) {
+                        logger.debug("failed to delete file [{}]", ex, file);
                     }
                 }
-            }
-            if (failureToRename != null) {
-                throw failureToRename;
+                for (String fileToRename : filesToRename) {
+                    // now, rename the files... and fail it it won't work
+                    store.renameFile(prefix + fileToRename, fileToRename);
+                }
             }
             // now write checksums
             store.writeChecksums(onGoingRecovery.checksums);

src/test/java/org/elasticsearch/test/store/mock/MockDirectoryHelper.java

@@ -61,12 +61,12 @@ public class MockDirectoryHelper {
     private final boolean noDeleteOpenFile;
 
     public MockDirectoryHelper(ShardId shardId, Settings indexSettings, ESLogger logger) {
+        final long seed = indexSettings.getAsLong(AbstractIntegrationTest.INDEX_SEED_SETTING, 0l);
+        random = new Random(seed);
         randomIOExceptionRate = indexSettings.getAsDouble(RANDOM_IO_EXCEPTION_RATE, 0.0d);
         randomIOExceptionRateOnOpen = indexSettings.getAsDouble(RANDOM_IO_EXCEPTION_RATE_ON_OPEN, 0.0d);
         preventDoubleWrite = indexSettings.getAsBoolean(RANDOM_PREVENT_DOUBLE_WRITE, true); // true is default in MDW
-        noDeleteOpenFile = indexSettings.getAsBoolean(RANDOM_NO_DELETE_OPEN_FILE, true); // true is default in MDW
-        final long seed = indexSettings.getAsLong(AbstractIntegrationTest.INDEX_SEED_SETTING, 0l);
-        random = new Random(seed);
+        noDeleteOpenFile = indexSettings.getAsBoolean(RANDOM_NO_DELETE_OPEN_FILE, random.nextBoolean()); // true is default in MDW
         random.nextInt(shardId.getId() + 1); // some randomness per shard
         throttle = Throttling.valueOf(indexSettings.get(RANDOM_THROTTLE, random.nextDouble() < 0.1 ? "SOMETIMES" : "NEVER"));
         checkIndexOnClose = indexSettings.getAsBoolean(CHECK_INDEX_ON_CLOSE, random.nextDouble() < 0.1);