Better failure for source-only snapshots of partially/fully mounted indices (#86207)

Taking a snapshot of a partially or fully mounted index in a source-only 
repository is not supported and stops the data node by throwing an 
AssertionError.

Instead of throwing such an error, this change simply fails the snapshot 
of searchable snapshot shards in source-only repository and report a 
better snapshot failure.

docs/changelog/86207.yaml

@@ -0,0 +1,5 @@
+pr: 86207
+summary: Better failure for source-only snapshots of partially/fully mounted indices
+area: Snapshot/Restore
+type: bug
+issues: []

x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotRepository.java

@@ -148,7 +148,15 @@ public final class SourceOnlySnapshotRepository extends FilterRepository {
         final Store store = context.store();
         Directory unwrap = FilterDirectory.unwrap(store.directory());
         if (unwrap instanceof FSDirectory == false) {
-            throw new AssertionError("expected FSDirectory but got " + unwrap.toString());
+            context.onFailure(
+                new IllegalStateException(
+                    context.indexCommit()
+                        + " is not a regular index, but ["
+                        + unwrap.toString()
+                        + "]  and cannot be snapshotted into a source-only repository"
+                )
+            );
+            return;
         }
         Path dataPath = ((FSDirectory) unwrap).getDirectory().getParent();
         // TODO should we have a snapshot tmp directory per shard that is maintained by the system?

x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsRestTestCase.java

@@ -49,6 +49,8 @@ import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Function;
 
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
+import static org.hamcrest.Matchers.allOf;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
@@ -246,6 +248,34 @@ public abstract class AbstractSearchableSnapshotsRestTestCase extends ESRestTest
                     randomFrom(Boolean.TRUE, Boolean.FALSE, null)
                 );
                 assertThat(extractValue(searchResults, "hits.total.value"), equalTo(numDocs));
+
+                // takes a snapshot of the searchable snapshot index into the source-only repository should fail
+                String sourceOnlySnapshot = "source-only-snap-" + randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
+                final Request request = new Request(HttpPut.METHOD_NAME, "_snapshot/" + WRITE_REPOSITORY_NAME + '/' + sourceOnlySnapshot);
+                request.addParameter("wait_for_completion", "true");
+                request.setJsonEntity("""
+                    {
+                        "include_global_state": false,
+                        "indices" : "%s"
+                    }
+                    """.formatted(indexName));
+
+                final Response response = adminClient().performRequest(request);
+                assertThat(response.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus()));
+
+                final List<Map<String, Object>> failures = extractValue(responseAsMap(response), "snapshot.failures");
+                assertThat(failures, notNullValue());
+                assertThat(failures.size(), greaterThan(0));
+                for (Map<String, Object> failure : failures) {
+                    assertThat(extractValue(failure, "status"), equalTo(RestStatus.INTERNAL_SERVER_ERROR.toString()));
+                    assertThat(
+                        extractValue(failure, "reason"),
+                        allOf(
+                            containsString("is not a regular index"),
+                            containsString("cannot be snapshotted into a source-only repository")
+                        )
+                    );
+                }
             }
         }, true);
     }