Add Partial snapshot state

Currently even if some shards of the snapshot are not snapshotted successfully, the snapshot is still marked as "SUCCESS". Users may miss the fact the there are shard failures present in the snapshot and think that snapshot was completed. This change adds a new snapshot state "PARTIAL" that provides a quick indication that the snapshot was only partially successful.

Closes #5792

src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java

@@ -206,6 +206,9 @@ public class TransportSnapshotsStatusAction extends TransportMasterNodeOperation
                                 state = SnapshotMetaData.State.FAILED;
                                 break;
                             case SUCCESS:
+                            case PARTIAL:
+                                // Translating both PARTIAL and SUCCESS to SUCCESS for now
+                                // TODO: add the differentiation on the metadata level in the next major release
                                 state = SnapshotMetaData.State.SUCCESS;
                                 break;
                             default:

src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java

@@ -312,7 +312,11 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent<Rep
             String blobName = snapshotBlobName(snapshotId);
             BlobStoreSnapshot.Builder updatedSnapshot = BlobStoreSnapshot.builder().snapshot(snapshot);
             if (failure == null) {
-                updatedSnapshot.success();
+                if (shardFailures.isEmpty()) {
+                    updatedSnapshot.success();
+                } else {
+                    updatedSnapshot.partial();
+                }
                 updatedSnapshot.failures(totalShards, shardFailures);
             } else {
                 updatedSnapshot.failed(failure);

src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreSnapshot.java

@@ -20,7 +20,6 @@
 package org.elasticsearch.repositories.blobstore;
 
 import com.google.common.collect.ImmutableList;
-import com.google.common.primitives.Longs;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -290,6 +289,16 @@ public class BlobStoreSnapshot implements Snapshot {
             return this;
         }
 
+        /**
+         * Marks snapshot as partially successful
+         *
+         * @return this builder
+         */
+        public Builder partial() {
+            this.state = SnapshotState.PARTIAL;
+            return this;
+        }
+
         /**
          * Marks snapshot as failed and saves failure reason
          *

src/main/java/org/elasticsearch/snapshots/RestoreService.java

@@ -119,7 +119,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
             final MetaData metaData = repository.readSnapshotMetaData(snapshotId, filteredIndices);
 
             // Make sure that we can restore from this snapshot
-            if (snapshot.state() != SnapshotState.SUCCESS) {
+            if (!snapshot.state().restorable()) {
                 throw new SnapshotRestoreException(snapshotId, "unsupported snapshot state [" + snapshot.state() + "]");
             }
             if (Version.CURRENT.before(snapshot.version())) {

src/main/java/org/elasticsearch/snapshots/SnapshotState.java

@@ -28,20 +28,30 @@ public enum SnapshotState {
     /**
      * Snapshot process has started
      */
-    IN_PROGRESS((byte) 0),
+    IN_PROGRESS((byte) 0, false, false),
     /**
      * Snapshot process completed successfully
      */
-    SUCCESS((byte) 1),
+    SUCCESS((byte) 1, true, true),
     /**
      * Snapshot failed
      */
-    FAILED((byte) 2);
+    FAILED((byte) 2, true, false),
+    /**
+     * Snapshot was partial successful
+     */
+    PARTIAL((byte) 3, true, true);
 
     private byte value;
 
-    private SnapshotState(byte value) {
+    private boolean completed;
+
+    private boolean restorable;
+
+    private SnapshotState(byte value, boolean completed, boolean restorable) {
         this.value = value;
+        this.completed = completed;
+        this.restorable = restorable;
     }
 
     /**
@@ -59,7 +69,17 @@ public enum SnapshotState {
      * @return true if snapshot completed, false otherwise
      */
     public boolean completed() {
-        return this == SUCCESS || this == FAILED;
+        return completed;
+    }
+
+
+    /**
+     * Returns true if snapshot can be restored (at least partially)
+     *
+     * @return true if snapshot can be restored, false otherwise
+     */
+    public boolean restorable() {
+        return restorable;
     }
 
     /**
@@ -76,6 +96,8 @@ public enum SnapshotState {
                 return SUCCESS;
             case 2:
                 return FAILED;
+            case 3:
+                return PARTIAL;
             default:
                 throw new ElasticsearchIllegalArgumentException("No snapshot state for value [" + value + "]");
         }

src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreTests.java

@@ -238,7 +238,7 @@ public class DedicatedClusterSnapshotRestoreTests extends AbstractSnapshotTests
         assertThat(createSnapshotResponse.getSnapshotInfo().totalShards(), equalTo(12));
         assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), lessThan(12));
         assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(6));
-        assertThat(client().admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap-2").execute().actionGet().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
+        assertThat(client().admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap-2").execute().actionGet().getSnapshots().get(0).state(), equalTo(SnapshotState.PARTIAL));
 
         assertAcked(client().admin().indices().prepareClose("test-idx-1", "test-idx-2").execute().actionGet());
 

src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreTests.java

@@ -409,6 +409,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
             GetSnapshotsResponse getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("test-repo").addSnapshots("test-snap").get();
             assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(1));
             SnapshotInfo snapshotInfo = getSnapshotsResponse.getSnapshots().get(0);
+            assertThat(snapshotInfo.state(), equalTo(SnapshotState.PARTIAL));
             assertThat(snapshotInfo.shardFailures().size(), greaterThan(0));
             assertThat(snapshotInfo.totalShards(), greaterThan(snapshotInfo.successfulShards()));