Support failure store in snapshot and restore operations (#107281)

In this PR we update the snapshot and restore implementations to include
the failure store of a data stream when a data stream is being snapshot.

- When a data stream is requested to be snapshot, then it implies all backing indices and failure stores.
- When a data stream is being restored, then it implies all backing indices and failure stores (assuming the feature flag is also enabled)
- When individual backing or failure store indices that have been removed are being restored they need to be manually added to the data stream. This functionality worked out of the box.

modules/data-streams/build.gradle

@@ -20,6 +20,7 @@ restResources {
 
 dependencies {
   testImplementation project(path: ':test:test-clusters')
+  internalClusterTestImplementation project(":modules:mapper-extras")
 }
 
 tasks.named('yamlRestTest') {

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java

@@ -1281,7 +1281,7 @@ public class DataStreamIT extends ESIntegTestCase {
     public void testGetDataStream() throws Exception {
         Settings settings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, maximumNumberOfReplicas() + 2).build();
         DataStreamLifecycle lifecycle = DataStreamLifecycle.newBuilder().dataRetention(randomMillisUpToYear9999()).build();
-        putComposableIndexTemplate("template_for_foo", null, List.of("metrics-foo*"), settings, null, null, lifecycle);
+        putComposableIndexTemplate("template_for_foo", null, List.of("metrics-foo*"), settings, null, null, lifecycle, false);
         int numDocsFoo = randomIntBetween(2, 16);
         indexDocs("metrics-foo", numDocsFoo);
 
@@ -1642,7 +1642,8 @@ public class DataStreamIT extends ESIntegTestCase {
                 null,
                 null,
                 Map.of("my-alias", AliasMetadata.builder("my-alias").build()),
-                null
+                null,
+                false
             );
             var request = new CreateDataStreamAction.Request("my-ds");
             assertAcked(client().execute(CreateDataStreamAction.INSTANCE, request).actionGet());
@@ -1675,7 +1676,8 @@ public class DataStreamIT extends ESIntegTestCase {
                 null,
                 null,
                 Map.of("logs", AliasMetadata.builder("logs").build()),
-                null
+                null,
+                false
             );
 
             var request = new CreateDataStreamAction.Request("logs-es");
@@ -1712,7 +1714,8 @@ public class DataStreamIT extends ESIntegTestCase {
                     null,
                     null,
                     Map.of("logs", AliasMetadata.builder("logs").build()),
-                    null
+                    null,
+                    false
                 )
             );
             assertThat(
@@ -1902,7 +1905,11 @@ public class DataStreamIT extends ESIntegTestCase {
     }
 
     public static void putComposableIndexTemplate(String id, List<String> patterns) throws IOException {
-        putComposableIndexTemplate(id, null, patterns, null, null);
+        putComposableIndexTemplate(id, patterns, false);
+    }
+
+    public static void putComposableIndexTemplate(String id, List<String> patterns, boolean withFailureStore) throws IOException {
+        putComposableIndexTemplate(id, null, patterns, null, null, null, null, withFailureStore);
     }
 
     public void testPartitionedTemplate() throws IOException {
@@ -2277,7 +2284,7 @@ public class DataStreamIT extends ESIntegTestCase {
         @Nullable Settings settings,
         @Nullable Map<String, Object> metadata
     ) throws IOException {
-        putComposableIndexTemplate(id, mappings, patterns, settings, metadata, null, null);
+        putComposableIndexTemplate(id, mappings, patterns, settings, metadata, null, null, false);
     }
 
     static void putComposableIndexTemplate(
@@ -2287,7 +2294,8 @@ public class DataStreamIT extends ESIntegTestCase {
         @Nullable Settings settings,
         @Nullable Map<String, Object> metadata,
         @Nullable Map<String, AliasMetadata> aliases,
-        @Nullable DataStreamLifecycle lifecycle
+        @Nullable DataStreamLifecycle lifecycle,
+        boolean withFailureStore
     ) throws IOException {
         TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request(id);
         request.indexTemplate(
@@ -2295,7 +2303,7 @@ public class DataStreamIT extends ESIntegTestCase {
                 .indexPatterns(patterns)
                 .template(new Template(settings, mappings == null ? null : CompressedXContent.fromJSON(mappings), aliases, lifecycle))
                 .metadata(metadata)
-                .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate())
+                .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false, withFailureStore))
                 .build()
         );
         client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet();

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java

@@ -36,6 +36,7 @@ import org.elasticsearch.cluster.metadata.DataStreamAlias;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.rest.RestStatus;
@@ -81,13 +82,17 @@ public class DataStreamsSnapshotsIT extends AbstractSnapshotIntegTestCase {
 
     private String dsBackingIndexName;
     private String otherDsBackingIndexName;
+    private String fsBackingIndexName;
+    private String fsFailureIndexName;
     private String ds2BackingIndexName;
     private String otherDs2BackingIndexName;
+    private String fs2BackingIndexName;
+    private String fs2FailureIndexName;
     private String id;
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return List.of(MockRepository.Plugin.class, DataStreamsPlugin.class);
+        return List.of(MockRepository.Plugin.class, DataStreamsPlugin.class, MapperExtrasPlugin.class);
     }
 
     @Before
@@ -97,6 +102,18 @@ public class DataStreamsSnapshotsIT extends AbstractSnapshotIntegTestCase {
         createRepository(REPO, "fs", location);
 
         DataStreamIT.putComposableIndexTemplate("t1", List.of("ds", "other-ds"));
+        DataStreamIT.putComposableIndexTemplate("t2", """
+            {
+                "properties": {
+                  "@timestamp": {
+                    "type": "date",
+                    "format": "date_optional_time||epoch_millis"
+                  },
+                  "flag": {
+                    "type": "boolean"
+                  }
+              }
+            }""", List.of("with-fs"), null, null, null, null, true);
 
         CreateDataStreamAction.Request request = new CreateDataStreamAction.Request("ds");
         AcknowledgedResponse response = client.execute(CreateDataStreamAction.INSTANCE, request).get();
@@ -106,15 +123,30 @@ public class DataStreamsSnapshotsIT extends AbstractSnapshotIntegTestCase {
         response = client.execute(CreateDataStreamAction.INSTANCE, request).get();
         assertTrue(response.isAcknowledged());
 
+        request = new CreateDataStreamAction.Request("with-fs");
+        response = client.execute(CreateDataStreamAction.INSTANCE, request).get();
+        assertTrue(response.isAcknowledged());
+
         // Resolve backing index names after data streams have been created:
         // (these names have a date component, and running around midnight could lead to test failures otherwise)
         GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { "*" });
         GetDataStreamAction.Response getDataStreamResponse = client.execute(GetDataStreamAction.INSTANCE, getDataStreamRequest).actionGet();
         dsBackingIndexName = getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices().get(0).getName();
         otherDsBackingIndexName = getDataStreamResponse.getDataStreams().get(1).getDataStream().getIndices().get(0).getName();
+        fsBackingIndexName = getDataStreamResponse.getDataStreams().get(2).getDataStream().getIndices().get(0).getName();
+        fsFailureIndexName = getDataStreamResponse.getDataStreams()
+            .get(2)
+            .getDataStream()
+            .getFailureIndices()
+            .getIndices()
+            .get(0)
+            .getName();
+
         // Will be used in some tests, to test renaming while restoring a snapshot:
         ds2BackingIndexName = dsBackingIndexName.replace("-ds-", "-ds2-");
         otherDs2BackingIndexName = otherDsBackingIndexName.replace("-other-ds-", "-other-ds2-");
+        fs2BackingIndexName = fsBackingIndexName.replace("-with-fs-", "-with-fs2-");
+        fs2FailureIndexName = fsFailureIndexName.replace("-with-fs-", "-with-fs2-");
 
         DocWriteResponse indexResponse = client.prepareIndex("ds")
             .setOpType(DocWriteRequest.OpType.CREATE)
@@ -232,12 +264,16 @@ public class DataStreamsSnapshotsIT extends AbstractSnapshotIntegTestCase {
         GetDataStreamAction.Response ds = client.execute(GetDataStreamAction.INSTANCE, getDataSteamRequest).get();
         assertThat(
             ds.getDataStreams().stream().map(e -> e.getDataStream().getName()).collect(Collectors.toList()),
-            contains(equalTo("ds"), equalTo("other-ds"))
+            contains(equalTo("ds"), equalTo("other-ds"), equalTo("with-fs"))
         );
         List<Index> backingIndices = ds.getDataStreams().get(0).getDataStream().getIndices();
         assertThat(backingIndices.stream().map(Index::getName).collect(Collectors.toList()), contains(dsBackingIndexName));
         backingIndices = ds.getDataStreams().get(1).getDataStream().getIndices();
         assertThat(backingIndices.stream().map(Index::getName).collect(Collectors.toList()), contains(otherDsBackingIndexName));
+        backingIndices = ds.getDataStreams().get(2).getDataStream().getIndices();
+        assertThat(backingIndices.stream().map(Index::getName).collect(Collectors.toList()), contains(fsBackingIndexName));
+        List<Index> failureIndices = ds.getDataStreams().get(2).getDataStream().getFailureIndices().getIndices();
+        assertThat(failureIndices.stream().map(Index::getName).collect(Collectors.toList()), contains(fsFailureIndexName));
     }
 
     public void testSnapshotAndRestoreInPlace() {
@@ -295,13 +331,72 @@ public class DataStreamsSnapshotsIT extends AbstractSnapshotIntegTestCase {
 
         // The backing index created as part of rollover should still exist (but just not part of the data stream)
         assertThat(indexExists(backingIndexAfterSnapshot), is(true));
-        // An additional rollover should create a new backing index (3th generation) and leave .ds-ds-...-2 index as is:
+        // An additional rollover should create a new backing index (3rd generation) and leave .ds-ds-...-2 index as is:
         rolloverRequest = new RolloverRequest("ds", null);
         rolloverResponse = client.admin().indices().rolloverIndex(rolloverRequest).actionGet();
         assertThat(rolloverResponse.isRolledOver(), is(true));
         assertThat(rolloverResponse.getNewIndex(), equalTo(DataStream.getDefaultBackingIndexName("ds", 3)));
     }
 
+    public void testFailureStoreSnapshotAndRestore() throws Exception {
+        CreateSnapshotResponse createSnapshotResponse = client.admin()
+            .cluster()
+            .prepareCreateSnapshot(REPO, SNAPSHOT)
+            .setWaitForCompletion(true)
+            .setIndices("with-fs")
+            .setIncludeGlobalState(false)
+            .get();
+
+        RestStatus status = createSnapshotResponse.getSnapshotInfo().status();
+        assertEquals(RestStatus.OK, status);
+
+        assertThat(getSnapshot(REPO, SNAPSHOT).indices(), containsInAnyOrder(fsBackingIndexName, fsFailureIndexName));
+
+        assertAcked(client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request("with-fs")));
+
+        {
+            RestoreSnapshotResponse restoreSnapshotResponse = client.admin()
+                .cluster()
+                .prepareRestoreSnapshot(REPO, SNAPSHOT)
+                .setWaitForCompletion(true)
+                .setIndices("with-fs")
+                .get();
+
+            assertEquals(2, restoreSnapshotResponse.getRestoreInfo().successfulShards());
+
+            GetDataStreamAction.Response ds = client.execute(
+                GetDataStreamAction.INSTANCE,
+                new GetDataStreamAction.Request(new String[] { "with-fs" })
+            ).get();
+            assertEquals(1, ds.getDataStreams().size());
+            assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size());
+            assertEquals(fsBackingIndexName, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName());
+            assertEquals(fsFailureIndexName, ds.getDataStreams().get(0).getDataStream().getFailureIndices().getIndices().get(0).getName());
+        }
+        {
+            // With rename pattern
+            RestoreSnapshotResponse restoreSnapshotResponse = client.admin()
+                .cluster()
+                .prepareRestoreSnapshot(REPO, SNAPSHOT)
+                .setWaitForCompletion(true)
+                .setIndices("with-fs")
+                .setRenamePattern("-fs")
+                .setRenameReplacement("-fs2")
+                .get();
+
+            assertEquals(2, restoreSnapshotResponse.getRestoreInfo().successfulShards());
+
+            GetDataStreamAction.Response ds = client.execute(
+                GetDataStreamAction.INSTANCE,
+                new GetDataStreamAction.Request(new String[] { "with-fs2" })
+            ).get();
+            assertEquals(1, ds.getDataStreams().size());
+            assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size());
+            assertEquals(fs2BackingIndexName, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName());
+            assertEquals(fs2FailureIndexName, ds.getDataStreams().get(0).getDataStream().getFailureIndices().getIndices().get(0).getName());
+        }
+    }
+
     public void testSnapshotAndRestoreAllIncludeSpecificDataStream() throws Exception {
         DocWriteResponse indexResponse = client.prepareIndex("other-ds")
             .setOpType(DocWriteRequest.OpType.CREATE)
@@ -338,10 +433,13 @@ public class DataStreamsSnapshotsIT extends AbstractSnapshotIntegTestCase {
         if (filterDuringSnapshotting) {
             assertThat(getSnapshot(REPO, SNAPSHOT).indices(), containsInAnyOrder(backingIndexName));
         } else {
-            assertThat(getSnapshot(REPO, SNAPSHOT).indices(), containsInAnyOrder(dsBackingIndexName, otherDsBackingIndexName));
+            assertThat(
+                getSnapshot(REPO, SNAPSHOT).indices(),
+                containsInAnyOrder(dsBackingIndexName, otherDsBackingIndexName, fsBackingIndexName, fsFailureIndexName)
+            );
         }
 
-        assertAcked(client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(new String[] { "*" })).get());
+        assertAcked(client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request("*")).get());
         assertAcked(client.admin().indices().prepareDelete("*").setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN));
 
         RestoreSnapshotRequest restoreSnapshotRequest = new RestoreSnapshotRequest(REPO, SNAPSHOT);
@@ -395,7 +493,10 @@ public class DataStreamsSnapshotsIT extends AbstractSnapshotIntegTestCase {
 
         RestStatus status = createSnapshotResponse.getSnapshotInfo().status();
         assertEquals(RestStatus.OK, status);
-        assertThat(getSnapshot(REPO, SNAPSHOT).indices(), containsInAnyOrder(dsBackingIndexName, otherDsBackingIndexName));
+        assertThat(
+            getSnapshot(REPO, SNAPSHOT).indices(),
+            containsInAnyOrder(dsBackingIndexName, otherDsBackingIndexName, fsBackingIndexName, fsFailureIndexName)
+        );
 
         assertAcked(client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(new String[] { "*" })).get());
         assertAcked(client.admin().indices().prepareDelete("*").setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN));
@@ -403,7 +504,7 @@ public class DataStreamsSnapshotsIT extends AbstractSnapshotIntegTestCase {
         var restoreSnapshotRequest = new RestoreSnapshotRequest(REPO, SNAPSHOT).waitForCompletion(true).includeGlobalState(false);
         RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().restoreSnapshot(restoreSnapshotRequest).actionGet();
 
-        assertEquals(2, restoreSnapshotResponse.getRestoreInfo().successfulShards());
+        assertEquals(4, restoreSnapshotResponse.getRestoreInfo().successfulShards());
 
         assertEquals(DOCUMENT_SOURCE, client.prepareGet(dsBackingIndexName, id).get().getSourceAsMap());
         assertResponse(client.prepareSearch("ds"), response -> {
@@ -416,10 +517,10 @@ public class DataStreamsSnapshotsIT extends AbstractSnapshotIntegTestCase {
             GetDataStreamAction.INSTANCE,
             new GetDataStreamAction.Request(new String[] { "*" })
         ).get();
-        assertEquals(2, ds.getDataStreams().size());
+        assertEquals(3, ds.getDataStreams().size());
         assertThat(
             ds.getDataStreams().stream().map(i -> i.getDataStream().getName()).collect(Collectors.toList()),
-            containsInAnyOrder("ds", "other-ds")
+            containsInAnyOrder("ds", "other-ds", "with-fs")
         );
 
         GetAliasesResponse getAliasesResponse = client.admin().indices().getAliases(new GetAliasesRequest("my-alias")).actionGet();
@@ -451,14 +552,17 @@ public class DataStreamsSnapshotsIT extends AbstractSnapshotIntegTestCase {
 
         RestStatus status = createSnapshotResponse.getSnapshotInfo().status();
         assertEquals(RestStatus.OK, status);
-        assertThat(getSnapshot(REPO, SNAPSHOT).indices(), containsInAnyOrder(dsBackingIndexName, otherDsBackingIndexName));
+        assertThat(
+            getSnapshot(REPO, SNAPSHOT).indices(),
+            containsInAnyOrder(dsBackingIndexName, otherDsBackingIndexName, fsBackingIndexName, fsFailureIndexName)
+        );
 
-        assertAcked(client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(new String[] { "*" })).get());
+        assertAcked(client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request("*")).get());
         assertAcked(client.admin().indices().prepareDelete("*").setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN));
 
         var restoreSnapshotRequest = new RestoreSnapshotRequest(REPO, SNAPSHOT).waitForCompletion(true).includeGlobalState(false);
         RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().restoreSnapshot(restoreSnapshotRequest).actionGet();
-        assertEquals(2, restoreSnapshotResponse.getRestoreInfo().successfulShards());
+        assertEquals(4, restoreSnapshotResponse.getRestoreInfo().successfulShards());
 
         assertEquals(DOCUMENT_SOURCE, client.prepareGet(dsBackingIndexName, id).get().getSourceAsMap());
         assertResponse(client.prepareSearch("ds"), response -> {
@@ -471,11 +575,15 @@ public class DataStreamsSnapshotsIT extends AbstractSnapshotIntegTestCase {
             GetDataStreamAction.INSTANCE,
             new GetDataStreamAction.Request(new String[] { "*" })
         ).get();
-        assertEquals(2, ds.getDataStreams().size());
+        assertEquals(3, ds.getDataStreams().size());
         assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size());
         assertEquals(dsBackingIndexName, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName());
         assertEquals(1, ds.getDataStreams().get(1).getDataStream().getIndices().size());
         assertEquals(otherDsBackingIndexName, ds.getDataStreams().get(1).getDataStream().getIndices().get(0).getName());
+        assertEquals(1, ds.getDataStreams().get(2).getDataStream().getIndices().size());
+        assertEquals(fsBackingIndexName, ds.getDataStreams().get(2).getDataStream().getIndices().get(0).getName());
+        assertEquals(1, ds.getDataStreams().get(2).getDataStream().getFailureIndices().getIndices().size());
+        assertEquals(fsFailureIndexName, ds.getDataStreams().get(2).getDataStream().getFailureIndices().getIndices().get(0).getName());
 
         GetAliasesResponse getAliasesResponse = client.admin().indices().getAliases(new GetAliasesRequest("my-alias")).actionGet();
         assertThat(getAliasesResponse.getDataStreamAliases().keySet(), containsInAnyOrder("ds", "other-ds"));
@@ -507,16 +615,19 @@ public class DataStreamsSnapshotsIT extends AbstractSnapshotIntegTestCase {
 
         RestStatus status = createSnapshotResponse.getSnapshotInfo().status();
         assertEquals(RestStatus.OK, status);
-        assertThat(getSnapshot(REPO, SNAPSHOT).indices(), containsInAnyOrder(dsBackingIndexName, otherDsBackingIndexName));
+        assertThat(
+            getSnapshot(REPO, SNAPSHOT).indices(),
+            containsInAnyOrder(dsBackingIndexName, otherDsBackingIndexName, fsBackingIndexName, fsFailureIndexName)
+        );
 
-        assertAcked(client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(new String[] { "*" })).get());
+        assertAcked(client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request("*")).get());
         assertAcked(client.admin().indices().prepareDelete("*").setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN));
 
         var restoreSnapshotRequest = new RestoreSnapshotRequest(REPO, SNAPSHOT).waitForCompletion(true)
             .includeGlobalState(false)
             .includeAliases(false);
         RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().restoreSnapshot(restoreSnapshotRequest).actionGet();
-        assertEquals(2, restoreSnapshotResponse.getRestoreInfo().successfulShards());
+        assertEquals(4, restoreSnapshotResponse.getRestoreInfo().successfulShards());
 
         assertEquals(DOCUMENT_SOURCE, client.prepareGet(dsBackingIndexName, id).get().getSourceAsMap());
         assertResponse(client.prepareSearch("ds"), response -> {
@@ -529,11 +640,15 @@ public class DataStreamsSnapshotsIT extends AbstractSnapshotIntegTestCase {
             GetDataStreamAction.INSTANCE,
             new GetDataStreamAction.Request(new String[] { "*" })
         ).get();
-        assertEquals(2, ds.getDataStreams().size());
+        assertEquals(3, ds.getDataStreams().size());
         assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size());
         assertEquals(dsBackingIndexName, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName());
         assertEquals(1, ds.getDataStreams().get(1).getDataStream().getIndices().size());
         assertEquals(otherDsBackingIndexName, ds.getDataStreams().get(1).getDataStream().getIndices().get(0).getName());
+        assertEquals(1, ds.getDataStreams().get(2).getDataStream().getIndices().size());
+        assertEquals(fsBackingIndexName, ds.getDataStreams().get(2).getDataStream().getIndices().get(0).getName());
+        assertEquals(1, ds.getDataStreams().get(2).getDataStream().getIndices().size());
+        assertEquals(fsFailureIndexName, ds.getDataStreams().get(2).getDataStream().getFailureIndices().getIndices().get(0).getName());
 
         GetAliasesResponse getAliasesResponse = client.admin().indices().getAliases(new GetAliasesRequest("*")).actionGet();
         assertThat(getAliasesResponse.getDataStreamAliases(), anEmptyMap());
@@ -930,7 +1045,32 @@ public class DataStreamsSnapshotsIT extends AbstractSnapshotIntegTestCase {
             .prepareRestoreSnapshot(REPO, snapshot)
             .setIndices(indexWithoutDataStream)
             .setWaitForCompletion(true)
-            .setRestoreGlobalState(randomBoolean())
+            .setRestoreGlobalState(false)
+            .get()
+            .getRestoreInfo();
+        assertThat(restoreInfo.failedShards(), is(0));
+        assertThat(restoreInfo.successfulShards(), is(1));
+    }
+
+    /**
+     * This test is a copy of the {@link #testPartialRestoreSnapshotThatIncludesDataStream()} the only difference
+     * is that one include the global state and one doesn't. In general this shouldn't matter that's why it used to be
+     * a random parameter of the test, but because of #107515 it fails when we include the global state. Keep them
+     * separate until this is fixed.
+     */
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/107515")
+    public void testPartialRestoreSnapshotThatIncludesDataStreamWithGlobalState() {
+        final String snapshot = "test-snapshot";
+        final String indexWithoutDataStream = "test-idx-no-ds";
+        createIndexWithContent(indexWithoutDataStream);
+        createFullSnapshot(REPO, snapshot);
+        assertAcked(client.admin().indices().prepareDelete(indexWithoutDataStream));
+        RestoreInfo restoreInfo = client.admin()
+            .cluster()
+            .prepareRestoreSnapshot(REPO, snapshot)
+            .setIndices(indexWithoutDataStream)
+            .setWaitForCompletion(true)
+            .setRestoreGlobalState(true)
             .get()
             .getRestoreInfo();
         assertThat(restoreInfo.failedShards(), is(0));
@@ -1027,7 +1167,32 @@ public class DataStreamsSnapshotsIT extends AbstractSnapshotIntegTestCase {
             .cluster()
             .prepareRestoreSnapshot(REPO, snapshot)
             .setWaitForCompletion(true)
-            .setRestoreGlobalState(randomBoolean())
+            .setRestoreGlobalState(false)
+            .get()
+            .getRestoreInfo();
+        assertThat(restoreInfo.failedShards(), is(0));
+        assertThat(restoreInfo.successfulShards(), is(1));
+    }
+
+    /**
+     * This test is a copy of the {@link #testExcludeDSFromSnapshotWhenExcludingItsIndices()} the only difference
+     * is that one include the global state and one doesn't. In general this shouldn't matter that's why it used to be
+     * a random parameter of the test, but because of #107515 it fails when we include the global state. Keep them
+     * separate until this is fixed.
+     */
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/107515")
+    public void testExcludeDSFromSnapshotWhenExcludingItsIndicesWithGlobalState() {
+        final String snapshot = "test-snapshot";
+        final String indexWithoutDataStream = "test-idx-no-ds";
+        createIndexWithContent(indexWithoutDataStream);
+        final SnapshotInfo snapshotInfo = createSnapshot(REPO, snapshot, List.of("*", "-.*"));
+        assertThat(snapshotInfo.dataStreams(), empty());
+        assertAcked(client.admin().indices().prepareDelete(indexWithoutDataStream));
+        RestoreInfo restoreInfo = client.admin()
+            .cluster()
+            .prepareRestoreSnapshot(REPO, snapshot)
+            .setWaitForCompletion(true)
+            .setRestoreGlobalState(true)
             .get()
             .getRestoreInfo();
         assertThat(restoreInfo.failedShards(), is(0));
@@ -1051,7 +1216,7 @@ public class DataStreamsSnapshotsIT extends AbstractSnapshotIntegTestCase {
         assertEquals(RestStatus.OK, restoreSnapshotResponse.status());
 
         GetDataStreamAction.Request getRequest = new GetDataStreamAction.Request(new String[] { "*" });
-        assertThat(client.execute(GetDataStreamAction.INSTANCE, getRequest).get().getDataStreams(), hasSize(2));
+        assertThat(client.execute(GetDataStreamAction.INSTANCE, getRequest).get().getDataStreams(), hasSize(3));
         assertNotNull(client.admin().indices().prepareGetIndex().setIndices(indexName).get());
     }
 

server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java

@@ -15,6 +15,7 @@ import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
+import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -65,7 +66,9 @@ public class CreateSnapshotRequest extends MasterNodeRequest<CreateSnapshotReque
 
     private String[] indices = EMPTY_ARRAY;
 
-    private IndicesOptions indicesOptions = IndicesOptions.strictExpandHidden();
+    private IndicesOptions indicesOptions = DataStream.isFailureStoreFeatureFlagEnabled()
+        ? IndicesOptions.strictExpandHiddenIncludeFailureStore()
+        : IndicesOptions.strictExpandHidden();
 
     private String[] featureStates = EMPTY_ARRAY;
 

server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java

@@ -13,6 +13,7 @@ import org.elasticsearch.TransportVersions;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
+import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -41,7 +42,9 @@ public class RestoreSnapshotRequest extends MasterNodeRequest<RestoreSnapshotReq
     private String snapshot;
     private String repository;
     private String[] indices = Strings.EMPTY_ARRAY;
-    private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpen();
+    private IndicesOptions indicesOptions = DataStream.isFailureStoreFeatureFlagEnabled()
+        ? IndicesOptions.strictExpandOpenIncludeFailureStore()
+        : IndicesOptions.strictExpandOpen();
     private String[] featureStates = Strings.EMPTY_ARRAY;
     private String renamePattern;
     private String renameReplacement;

server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java

@@ -570,6 +570,25 @@ public record IndicesOptions(
         )
         .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(false))
         .build();
+    public static final IndicesOptions STRICT_EXPAND_OPEN_FAILURE_STORE = IndicesOptions.builder()
+        .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
+        .wildcardOptions(
+            WildcardOptions.builder()
+                .matchOpen(true)
+                .matchClosed(false)
+                .includeHidden(false)
+                .allowEmptyExpressions(true)
+                .resolveAliases(true)
+        )
+        .gatekeeperOptions(
+            GatekeeperOptions.builder()
+                .allowAliasToMultipleIndices(true)
+                .allowClosedIndices(true)
+                .allowFailureIndices(true)
+                .ignoreThrottled(false)
+        )
+        .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(true))
+        .build();
     public static final IndicesOptions LENIENT_EXPAND_OPEN = IndicesOptions.builder()
         .concreteTargetOptions(ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS)
         .wildcardOptions(
@@ -674,6 +693,58 @@ public record IndicesOptions(
         )
         .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(false))
         .build();
+    public static final IndicesOptions LENIENT_EXPAND_OPEN_CLOSED_FAILURE_STORE = IndicesOptions.builder()
+        .concreteTargetOptions(ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS)
+        .wildcardOptions(
+            WildcardOptions.builder()
+                .matchOpen(true)
+                .matchClosed(true)
+                .includeHidden(false)
+                .allowEmptyExpressions(true)
+                .resolveAliases(true)
+        )
+        .gatekeeperOptions(
+            GatekeeperOptions.builder()
+                .allowAliasToMultipleIndices(true)
+                .allowClosedIndices(true)
+                .allowFailureIndices(true)
+                .ignoreThrottled(false)
+        )
+        .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(true))
+        .build();
+    public static final IndicesOptions STRICT_EXPAND_OPEN_CLOSED_HIDDEN_FAILURE_STORE = IndicesOptions.builder()
+        .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
+        .wildcardOptions(
+            WildcardOptions.builder().matchOpen(true).matchClosed(true).includeHidden(true).allowEmptyExpressions(true).resolveAliases(true)
+        )
+        .gatekeeperOptions(
+            GatekeeperOptions.builder()
+                .allowAliasToMultipleIndices(true)
+                .allowClosedIndices(true)
+                .allowFailureIndices(true)
+                .ignoreThrottled(false)
+        )
+        .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(true))
+        .build();
+    public static final IndicesOptions STRICT_EXPAND_OPEN_CLOSED_FAILURE_STORE = IndicesOptions.builder()
+        .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
+        .wildcardOptions(
+            WildcardOptions.builder()
+                .matchOpen(true)
+                .matchClosed(true)
+                .includeHidden(false)
+                .allowEmptyExpressions(true)
+                .resolveAliases(true)
+        )
+        .gatekeeperOptions(
+            GatekeeperOptions.builder()
+                .allowAliasToMultipleIndices(true)
+                .allowClosedIndices(true)
+                .allowFailureIndices(true)
+                .ignoreThrottled(false)
+        )
+        .failureStoreOptions(FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(true))
+        .build();
     public static final IndicesOptions STRICT_EXPAND_OPEN_FORBID_CLOSED = IndicesOptions.builder()
         .concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
         .wildcardOptions(
@@ -1321,6 +1392,14 @@ public record IndicesOptions(
         return STRICT_EXPAND_OPEN;
     }
 
+    /**
+     * @return indices options that requires every specified index to exist, expands wildcards only to open indices and
+     * allows that no indices are resolved from wildcard expressions (not returning an error).
+     */
+    public static IndicesOptions strictExpandOpenIncludeFailureStore() {
+        return STRICT_EXPAND_OPEN_FAILURE_STORE;
+    }
+
     /**
      * @return indices options that requires every specified index to exist, expands wildcards only to open indices,
      * allows that no indices are resolved from wildcard expressions (not returning an error) and forbids the
@@ -1355,6 +1434,24 @@ public record IndicesOptions(
         return STRICT_EXPAND_OPEN_CLOSED_HIDDEN;
     }
 
+    /**
+     * @return indices option that expands wildcards to both open and closed indices, includes failure store
+     * (with data stream) and allows that indices can be missing and no indices are resolved from wildcard expressions
+     * (not returning an error).
+     */
+    public static IndicesOptions lenientExpandIncludeFailureStore() {
+        return LENIENT_EXPAND_OPEN_CLOSED_FAILURE_STORE;
+    }
+
+    /**
+     * @return indices option that requires every specified index to exist, expands wildcards to both open and closed indices, includes
+     * hidden indices, includes failure store (with data stream) and allows that no indices are resolved from wildcard expressions
+     * (not returning an error).
+     */
+    public static IndicesOptions strictExpandHiddenIncludeFailureStore() {
+        return STRICT_EXPAND_OPEN_CLOSED_HIDDEN_FAILURE_STORE;
+    }
+
     /**
      * @return indices option that requires each specified index or alias to exist, doesn't expand wildcards.
      */

server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreDefinition.java

@@ -57,7 +57,7 @@ public class DataStreamFailureStoreDefinition {
              *       "error": {
              *         "properties": {
              *           "message": {
-             *              "type": "wildcard"
+             *              "type": "match_only_text"
              *           },
              *           "stack_trace": {
              *              "type": "text"
@@ -109,7 +109,7 @@ public class DataStreamFailureStoreDefinition {
                     .startObject("error")
                     .startObject("properties")
                     .startObject("message")
-                    .field("type", "wildcard")
+                    .field("type", "match_only_text")
                     .endObject()
                     .startObject("stack_trace")
                     .field("type", "text")

server/src/main/java/org/elasticsearch/snapshots/RestoreService.java

@@ -388,18 +388,28 @@ public final class RestoreService implements ClusterStateApplier {
         // Remove the data streams from the list of requested indices
         requestIndices.removeAll(dataStreamsToRestore.keySet());
 
-        // And add the backing indices
-        final Set<String> nonSystemDataStreamIndices;
+        // And add the backing indices and failure indices of data streams (the distinction is important for renaming)
         final Set<String> systemDataStreamIndices;
+        final Set<String> nonSystemDataStreamBackingIndices;
+        final Set<String> nonSystemDataStreamFailureIndices;
         {
-            Map<Boolean, Set<String>> dataStreamIndices = dataStreamsToRestore.values()
+            Map<Boolean, Set<String>> backingIndices = dataStreamsToRestore.values()
                 .stream()
                 .flatMap(ds -> ds.getIndices().stream().map(idx -> new Tuple<>(ds.isSystem(), idx.getName())))
                 .collect(Collectors.partitioningBy(Tuple::v1, Collectors.mapping(Tuple::v2, Collectors.toSet())));
-            systemDataStreamIndices = dataStreamIndices.get(true);
-            nonSystemDataStreamIndices = dataStreamIndices.get(false);
+            Map<Boolean, Set<String>> failureIndices = Map.of();
+            if (DataStream.isFailureStoreFeatureFlagEnabled()) {
+                failureIndices = dataStreamsToRestore.values()
+                    .stream()
+                    .flatMap(ds -> ds.getFailureIndices().getIndices().stream().map(idx -> new Tuple<>(ds.isSystem(), idx.getName())))
+                    .collect(Collectors.partitioningBy(Tuple::v1, Collectors.mapping(Tuple::v2, Collectors.toSet())));
+            }
+            systemDataStreamIndices = Sets.union(backingIndices.get(true), failureIndices.get(true));
+            nonSystemDataStreamBackingIndices = backingIndices.get(false);
+            nonSystemDataStreamFailureIndices = failureIndices.get(false);
         }
-        requestIndices.addAll(nonSystemDataStreamIndices);
+        requestIndices.addAll(nonSystemDataStreamBackingIndices);
+        requestIndices.addAll(nonSystemDataStreamFailureIndices);
         final Set<String> allSystemIndicesToRestore = Stream.of(systemDataStreamIndices, featureStateIndices)
             .flatMap(Collection::stream)
             .collect(Collectors.toSet());
@@ -472,7 +482,8 @@ public final class RestoreService implements ClusterStateApplier {
                 renamedIndices(
                     request,
                     requestedIndicesIncludingSystem,
-                    nonSystemDataStreamIndices,
+                    nonSystemDataStreamBackingIndices,
+                    nonSystemDataStreamFailureIndices,
                     allSystemIndicesToRestore,
                     repositoryData
                 ),
@@ -555,7 +566,7 @@ public final class RestoreService implements ClusterStateApplier {
         List<String> requestedDataStreams = filterIndices(
             snapshotInfo.dataStreams(),
             Stream.of(requestIndices, featureStateDataStreams).flatMap(Collection::stream).toArray(String[]::new),
-            IndicesOptions.fromOptions(true, true, true, true)
+            IndicesOptions.lenientExpand()
         );
         if (requestedDataStreams.isEmpty()) {
             dataStreams = Map.of();
@@ -702,11 +713,19 @@ public final class RestoreService implements ClusterStateApplier {
         }
         List<Index> updatedIndices = dataStream.getIndices()
             .stream()
-            .map(i -> metadata.get(renameIndex(i.getName(), request, true)).getIndex())
+            .map(i -> metadata.get(renameIndex(i.getName(), request, true, false)).getIndex())
             .toList();
+        List<Index> updatedFailureIndices = DataStream.isFailureStoreFeatureFlagEnabled()
+            ? dataStream.getFailureIndices()
+                .getIndices()
+                .stream()
+                .map(i -> metadata.get(renameIndex(i.getName(), request, false, true)).getIndex())
+                .toList()
+            : List.of();
         return dataStream.copy()
             .setName(dataStreamName)
             .setBackingIndices(dataStream.getBackingIndices().copy().setIndices(updatedIndices).build())
+            .setFailureIndices(dataStream.getFailureIndices().copy().setIndices(updatedFailureIndices).build())
             .build();
     }
 
@@ -900,10 +919,33 @@ public final class RestoreService implements ClusterStateApplier {
         return failedShards;
     }
 
+    private static String renameIndex(String index, RestoreSnapshotRequest request, boolean isBackingIndex, boolean isFailureStore) {
+        if (request.renameReplacement() == null || request.renamePattern() == null) {
+            return index;
+        }
+        String prefix = null;
+        if (isBackingIndex && index.startsWith(DataStream.BACKING_INDEX_PREFIX)) {
+            prefix = DataStream.BACKING_INDEX_PREFIX;
+        }
+        if (isFailureStore && index.startsWith(DataStream.FAILURE_STORE_PREFIX)) {
+            prefix = DataStream.FAILURE_STORE_PREFIX;
+        }
+        String renamedIndex;
+        if (prefix != null) {
+            index = index.substring(prefix.length());
+        }
+        renamedIndex = index.replaceAll(request.renamePattern(), request.renameReplacement());
+        if (prefix != null) {
+            renamedIndex = prefix + renamedIndex;
+        }
+        return renamedIndex;
+    }
+
     private static Map<String, IndexId> renamedIndices(
         RestoreSnapshotRequest request,
         List<String> filteredIndices,
-        Set<String> dataStreamIndices,
+        Set<String> dataStreamBackingIndices,
+        Set<String> dataStreamFailureIndices,
         Set<String> featureIndices,
         RepositoryData repositoryData
     ) {
@@ -914,7 +956,12 @@ public final class RestoreService implements ClusterStateApplier {
                 // Don't rename system indices
                 renamedIndex = index;
             } else {
-                renamedIndex = renameIndex(index, request, dataStreamIndices.contains(index));
+                renamedIndex = renameIndex(
+                    index,
+                    request,
+                    dataStreamBackingIndices.contains(index),
+                    dataStreamFailureIndices.contains(index)
+                );
             }
             IndexId previousIndex = renamedIndices.put(renamedIndex, repositoryData.resolveIndexId(index));
             if (previousIndex != null) {
@@ -928,21 +975,6 @@ public final class RestoreService implements ClusterStateApplier {
         return Collections.unmodifiableMap(renamedIndices);
     }
 
-    private static String renameIndex(String index, RestoreSnapshotRequest request, boolean partOfDataStream) {
-        String renamedIndex = index;
-        if (request.renameReplacement() != null && request.renamePattern() != null) {
-            partOfDataStream = partOfDataStream && index.startsWith(DataStream.BACKING_INDEX_PREFIX);
-            if (partOfDataStream) {
-                index = index.substring(DataStream.BACKING_INDEX_PREFIX.length());
-            }
-            renamedIndex = index.replaceAll(request.renamePattern(), request.renameReplacement());
-            if (partOfDataStream) {
-                renamedIndex = DataStream.BACKING_INDEX_PREFIX + renamedIndex;
-            }
-        }
-        return renamedIndex;
-    }
-
     /**
      * Checks that snapshots can be restored and have compatible version
      * @param repository      repository name

server/src/main/java/org/elasticsearch/snapshots/SnapshotUtils.java

@@ -27,7 +27,8 @@ public class SnapshotUtils {
      *
      * @param availableIndices list of available indices
      * @param selectedIndices  list of selected indices
-     * @param indicesOptions    ignore indices flag
+     * @param indicesOptions   from the indices options it only uses {@link IndicesOptions.ConcreteTargetOptions#allowUnavailableTargets()}
+     *                         and {@link IndicesOptions.WildcardOptions#allowEmptyExpressions()}
      * @return filtered out indices
      */
     public static List<String> filterIndices(List<String> availableIndices, String[] selectedIndices, IndicesOptions indicesOptions) {

server/src/test/java/org/elasticsearch/snapshots/RestoreServiceTests.java

@@ -47,77 +47,112 @@ import static org.mockito.Mockito.when;
 public class RestoreServiceTests extends ESTestCase {
 
     public void testUpdateDataStream() {
+        long now = System.currentTimeMillis();
         String dataStreamName = "data-stream-1";
         String backingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1);
-        List<Index> indices = Collections.singletonList(new Index(backingIndexName, "uuid"));
+        List<Index> indices = List.of(new Index(backingIndexName, randomUUID()));
+        String failureIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, 1, now);
+        List<Index> failureIndices = List.of(new Index(failureIndexName, randomUUID()));
 
-        DataStream dataStream = DataStreamTestHelper.newInstance(dataStreamName, indices);
+        DataStream dataStream = DataStreamTestHelper.newInstance(dataStreamName, indices, failureIndices);
 
         Metadata.Builder metadata = mock(Metadata.Builder.class);
-        IndexMetadata indexMetadata = mock(IndexMetadata.class);
-        when(metadata.get(eq(backingIndexName))).thenReturn(indexMetadata);
-        Index updatedIndex = new Index(backingIndexName, "uuid2");
-        when(indexMetadata.getIndex()).thenReturn(updatedIndex);
+
+        IndexMetadata backingIndexMetadata = mock(IndexMetadata.class);
+        when(metadata.get(eq(backingIndexName))).thenReturn(backingIndexMetadata);
+        Index updatedBackingIndex = new Index(backingIndexName, randomUUID());
+        when(backingIndexMetadata.getIndex()).thenReturn(updatedBackingIndex);
+
+        IndexMetadata failureIndexMetadata = mock(IndexMetadata.class);
+        when(metadata.get(eq(failureIndexName))).thenReturn(failureIndexMetadata);
+        Index updatedFailureIndex = new Index(failureIndexName, randomUUID());
+        when(failureIndexMetadata.getIndex()).thenReturn(updatedFailureIndex);
 
         RestoreSnapshotRequest request = new RestoreSnapshotRequest();
 
         DataStream updateDataStream = RestoreService.updateDataStream(dataStream, metadata, request);
 
         assertEquals(dataStreamName, updateDataStream.getName());
-        assertEquals(Collections.singletonList(updatedIndex), updateDataStream.getIndices());
+        assertEquals(List.of(updatedBackingIndex), updateDataStream.getIndices());
+        assertEquals(List.of(updatedFailureIndex), updateDataStream.getFailureIndices().getIndices());
     }
 
     public void testUpdateDataStreamRename() {
+        long now = System.currentTimeMillis();
         String dataStreamName = "data-stream-1";
         String renamedDataStreamName = "data-stream-2";
         String backingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1);
         String renamedBackingIndexName = DataStream.getDefaultBackingIndexName(renamedDataStreamName, 1);
-        List<Index> indices = Collections.singletonList(new Index(backingIndexName, "uuid"));
+        List<Index> indices = List.of(new Index(backingIndexName, randomUUID()));
 
-        DataStream dataStream = DataStreamTestHelper.newInstance(dataStreamName, indices);
+        String failureIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, 1, now);
+        String renamedFailureIndexName = DataStream.getDefaultFailureStoreName(renamedDataStreamName, 1, now);
+        List<Index> failureIndices = List.of(new Index(failureIndexName, randomUUID()));
+
+        DataStream dataStream = DataStreamTestHelper.newInstance(dataStreamName, indices, failureIndices);
 
         Metadata.Builder metadata = mock(Metadata.Builder.class);
-        IndexMetadata indexMetadata = mock(IndexMetadata.class);
-        when(metadata.get(eq(renamedBackingIndexName))).thenReturn(indexMetadata);
-        Index renamedIndex = new Index(renamedBackingIndexName, "uuid2");
-        when(indexMetadata.getIndex()).thenReturn(renamedIndex);
+
+        IndexMetadata backingIndexMetadata = mock(IndexMetadata.class);
+        when(metadata.get(eq(renamedBackingIndexName))).thenReturn(backingIndexMetadata);
+        Index renamedBackingIndex = new Index(renamedBackingIndexName, randomUUID());
+        when(backingIndexMetadata.getIndex()).thenReturn(renamedBackingIndex);
+
+        IndexMetadata failureIndexMetadata = mock(IndexMetadata.class);
+        when(metadata.get(eq(renamedFailureIndexName))).thenReturn(failureIndexMetadata);
+        Index renamedFailureIndex = new Index(renamedFailureIndexName, randomUUID());
+        when(failureIndexMetadata.getIndex()).thenReturn(renamedFailureIndex);
 
         RestoreSnapshotRequest request = new RestoreSnapshotRequest().renamePattern("data-stream-1").renameReplacement("data-stream-2");
 
         DataStream renamedDataStream = RestoreService.updateDataStream(dataStream, metadata, request);
 
         assertEquals(renamedDataStreamName, renamedDataStream.getName());
-        assertEquals(Collections.singletonList(renamedIndex), renamedDataStream.getIndices());
+        assertEquals(List.of(renamedBackingIndex), renamedDataStream.getIndices());
+        assertEquals(List.of(renamedFailureIndex), renamedDataStream.getFailureIndices().getIndices());
     }
 
     public void testPrefixNotChanged() {
+        long now = System.currentTimeMillis();
         String dataStreamName = "ds-000001";
         String renamedDataStreamName = "ds2-000001";
         String backingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1);
         String renamedBackingIndexName = DataStream.getDefaultBackingIndexName(renamedDataStreamName, 1);
-        List<Index> indices = Collections.singletonList(new Index(backingIndexName, "uuid"));
+        List<Index> indices = Collections.singletonList(new Index(backingIndexName, randomUUID()));
 
-        DataStream dataStream = DataStreamTestHelper.newInstance(dataStreamName, indices);
+        String failureIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, 1, now);
+        String renamedFailureIndexName = DataStream.getDefaultFailureStoreName(renamedDataStreamName, 1, now);
+        List<Index> failureIndices = Collections.singletonList(new Index(failureIndexName, randomUUID()));
+
+        DataStream dataStream = DataStreamTestHelper.newInstance(dataStreamName, indices, failureIndices);
 
         Metadata.Builder metadata = mock(Metadata.Builder.class);
+
         IndexMetadata indexMetadata = mock(IndexMetadata.class);
         when(metadata.get(eq(renamedBackingIndexName))).thenReturn(indexMetadata);
-        Index renamedIndex = new Index(renamedBackingIndexName, "uuid2");
+        Index renamedIndex = new Index(renamedBackingIndexName, randomUUID());
         when(indexMetadata.getIndex()).thenReturn(renamedIndex);
 
+        IndexMetadata failureIndexMetadata = mock(IndexMetadata.class);
+        when(metadata.get(eq(renamedFailureIndexName))).thenReturn(failureIndexMetadata);
+        Index renamedFailureIndex = new Index(renamedFailureIndexName, randomUUID());
+        when(failureIndexMetadata.getIndex()).thenReturn(renamedFailureIndex);
+
         RestoreSnapshotRequest request = new RestoreSnapshotRequest().renamePattern("ds-").renameReplacement("ds2-");
 
         DataStream renamedDataStream = RestoreService.updateDataStream(dataStream, metadata, request);
 
         assertEquals(renamedDataStreamName, renamedDataStream.getName());
-        assertEquals(Collections.singletonList(renamedIndex), renamedDataStream.getIndices());
+        assertEquals(List.of(renamedIndex), renamedDataStream.getIndices());
+        assertEquals(List.of(renamedFailureIndex), renamedDataStream.getFailureIndices().getIndices());
 
         request = new RestoreSnapshotRequest().renamePattern("ds-000001").renameReplacement("ds2-000001");
 
         renamedDataStream = RestoreService.updateDataStream(dataStream, metadata, request);
 
         assertEquals(renamedDataStreamName, renamedDataStream.getName());
-        assertEquals(Collections.singletonList(renamedIndex), renamedDataStream.getIndices());
+        assertEquals(List.of(renamedIndex), renamedDataStream.getIndices());
+        assertEquals(List.of(renamedFailureIndex), renamedDataStream.getFailureIndices().getIndices());
     }
 
     public void testRefreshRepositoryUuidsDoesNothingIfDisabled() {