Make modify data stream API project aware (MP-1926)

Makes the modify data stream API (for changing the backing and failure
indices) project aware.

modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/ModifyDataStreamsTransportAction.java

@@ -12,12 +12,13 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.datastreams.ModifyDataStreamsAction;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
-import org.elasticsearch.action.support.master.AcknowledgedTransportMasterNodeAction;
-import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.action.support.master.AcknowledgedTransportMasterNodeProjectAction;
+import org.elasticsearch.cluster.ProjectState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetadataDataStreamsService;
+import org.elasticsearch.cluster.project.ProjectResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.injection.guice.Inject;
@@ -25,7 +26,7 @@ import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
-public class ModifyDataStreamsTransportAction extends AcknowledgedTransportMasterNodeAction<ModifyDataStreamsAction.Request> {
+public class ModifyDataStreamsTransportAction extends AcknowledgedTransportMasterNodeProjectAction<ModifyDataStreamsAction.Request> {
 
     private final MetadataDataStreamsService metadataDataStreamsService;
 
@@ -35,6 +36,7 @@ public class ModifyDataStreamsTransportAction extends AcknowledgedTransportMaste
         ClusterService clusterService,
         ThreadPool threadPool,
         ActionFilters actionFilters,
+        ProjectResolver projectResolver,
         IndexNameExpressionResolver indexNameExpressionResolver,
         MetadataDataStreamsService metadataDataStreamsService
     ) {
@@ -45,6 +47,7 @@ public class ModifyDataStreamsTransportAction extends AcknowledgedTransportMaste
             threadPool,
             actionFilters,
             ModifyDataStreamsAction.Request::new,
+            projectResolver,
             indexNameExpressionResolver,
             EsExecutors.DIRECT_EXECUTOR_SERVICE
         );
@@ -55,20 +58,24 @@ public class ModifyDataStreamsTransportAction extends AcknowledgedTransportMaste
     protected void masterOperation(
         Task task,
         ModifyDataStreamsAction.Request request,
-        ClusterState state,
+        ProjectState state,
         ActionListener<AcknowledgedResponse> listener
-    ) throws Exception {
-        metadataDataStreamsService.modifyDataStream(request, listener);
+    ) {
+        metadataDataStreamsService.modifyDataStream(state.projectId(), request, listener);
     }
 
     @Override
-    protected ClusterBlockException checkBlock(ModifyDataStreamsAction.Request request, ClusterState state) {
+    protected ClusterBlockException checkBlock(ModifyDataStreamsAction.Request request, ProjectState state) {
         ClusterBlockException globalBlock = state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
         if (globalBlock != null) {
             return globalBlock;
         }
         return state.blocks()
-            .indicesBlockedException(ClusterBlockLevel.METADATA_WRITE, indexNameExpressionResolver.concreteIndexNames(state, request));
+            .indicesBlockedException(
+                state.projectId(),
+                ClusterBlockLevel.METADATA_WRITE,
+                indexNameExpressionResolver.concreteIndexNames(state.metadata(), request)
+            );
     }
 
 }

modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/downsampling/DeleteSourceAndAddDownsampleToDS.java

@@ -92,7 +92,7 @@ public class DeleteSourceAndAddDownsampleToDS implements ClusterStateTaskListene
                     dataStreamName
                 );
                 Metadata.Builder newMetaData = Metadata.builder(state.metadata())
-                    .put(dataStream.addBackingIndex(state.metadata(), downsampleIndexMeta.getIndex()));
+                    .put(dataStream.addBackingIndex(state.metadata().getProject(), downsampleIndexMeta.getIndex()));
                 return ClusterState.builder(state).metadata(newMetaData).build();
             }
         } else {
@@ -161,7 +161,7 @@ public class DeleteSourceAndAddDownsampleToDS implements ClusterStateTaskListene
 
         newMetaData.put(updatedDownsampleMetadata, true);
         // we deleted the source already so let's add the downsample index to the data stream
-        newMetaData.put(dataStream.addBackingIndex(state.metadata(), downsampleIndexMeta.getIndex()));
+        newMetaData.put(dataStream.addBackingIndex(state.metadata().getProject(), downsampleIndexMeta.getIndex()));
         return ClusterState.builder(state).metadata(newMetaData).build();
     }
 

modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java

@@ -726,7 +726,7 @@ public class DataStreamLifecycleServiceTests extends ESTestCase {
         builder = Metadata.builder(clusterService.state().metadata()).put(newIndexMetadata, true);
         state = ClusterState.builder(clusterService.state()).metadata(builder).build();
         setState(clusterService, state);
-        DataStream dataStream2 = dataStream.addBackingIndex(clusterService.state().metadata(), newIndexMetadata.getIndex());
+        DataStream dataStream2 = dataStream.addBackingIndex(clusterService.state().metadata().getProject(), newIndexMetadata.getIndex());
         builder = Metadata.builder(clusterService.state().metadata());
         builder.put(dataStream2);
         state = ClusterState.builder(clusterService.state()).metadata(builder).build();
@@ -970,7 +970,7 @@ public class DataStreamLifecycleServiceTests extends ESTestCase {
         builder = Metadata.builder(clusterService.state().metadata()).put(newIndexMetadata, true);
         ClusterState state = ClusterState.builder(clusterService.state()).metadata(builder).build();
         setState(clusterService, state);
-        DataStream dataStream2 = dataStream.addBackingIndex(clusterService.state().metadata(), newIndexMetadata.getIndex());
+        DataStream dataStream2 = dataStream.addBackingIndex(clusterService.state().metadata().getProject(), newIndexMetadata.getIndex());
         builder = Metadata.builder(clusterService.state().metadata());
         builder.put(dataStream2);
         state = ClusterState.builder(clusterService.state()).metadata(builder).build();
@@ -1199,7 +1199,10 @@ public class DataStreamLifecycleServiceTests extends ESTestCase {
         builder = Metadata.builder(clusterService.state().metadata()).put(newIndexMetadata, true);
         state = ClusterState.builder(clusterService.state()).metadata(builder).build();
         setState(clusterService, state);
-        DataStream modifiedDataStream = dataStream.addBackingIndex(clusterService.state().metadata(), newIndexMetadata.getIndex());
+        DataStream modifiedDataStream = dataStream.addBackingIndex(
+            clusterService.state().metadata().getProject(),
+            newIndexMetadata.getIndex()
+        );
         builder = Metadata.builder(clusterService.state().metadata());
         builder.put(modifiedDataStream);
         state = ClusterState.builder(clusterService.state()).metadata(builder).build();

server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java

@@ -771,16 +771,16 @@ public final class DataStream implements SimpleDiffable<DataStream>, ToXContentO
      * @return new {@code DataStream} instance with the added backing index
      * @throws IllegalArgumentException if {@code index} is ineligible to be a backing index for the data stream
      */
-    public DataStream addBackingIndex(Metadata clusterMetadata, Index index) {
+    public DataStream addBackingIndex(ProjectMetadata project, Index index) {
         // validate that index is not part of another data stream
-        final var parentDataStream = clusterMetadata.getProject().getIndicesLookup().get(index.getName()).getParentDataStream();
+        final var parentDataStream = project.getIndicesLookup().get(index.getName()).getParentDataStream();
         if (parentDataStream != null) {
             validateDataStreamAlreadyContainsIndex(index, parentDataStream, false);
             return this;
         }
 
         // ensure that no aliases reference index
-        ensureNoAliasesOnIndex(clusterMetadata, index);
+        ensureNoAliasesOnIndex(project, index);
 
         List<Index> backingIndices = new ArrayList<>(this.backingIndices.indices);
         backingIndices.add(0, index);
@@ -798,15 +798,15 @@ public final class DataStream implements SimpleDiffable<DataStream>, ToXContentO
      * @return new {@code DataStream} instance with the added failure store index
      * @throws IllegalArgumentException if {@code index} is ineligible to be a failure store index for the data stream
      */
-    public DataStream addFailureStoreIndex(Metadata clusterMetadata, Index index) {
+    public DataStream addFailureStoreIndex(ProjectMetadata project, Index index) {
         // validate that index is not part of another data stream
-        final var parentDataStream = clusterMetadata.getProject().getIndicesLookup().get(index.getName()).getParentDataStream();
+        final var parentDataStream = project.getIndicesLookup().get(index.getName()).getParentDataStream();
         if (parentDataStream != null) {
             validateDataStreamAlreadyContainsIndex(index, parentDataStream, true);
             return this;
         }
 
-        ensureNoAliasesOnIndex(clusterMetadata, index);
+        ensureNoAliasesOnIndex(project, index);
 
         List<Index> updatedFailureIndices = new ArrayList<>(failureIndices.indices);
         updatedFailureIndices.add(0, index);
@@ -840,9 +840,8 @@ public final class DataStream implements SimpleDiffable<DataStream>, ToXContentO
         }
     }
 
-    private void ensureNoAliasesOnIndex(Metadata clusterMetadata, Index index) {
-        IndexMetadata im = clusterMetadata.getProject()
-            .index(clusterMetadata.getProject().getIndicesLookup().get(index.getName()).getWriteIndex());
+    private void ensureNoAliasesOnIndex(ProjectMetadata project, Index index) {
+        IndexMetadata im = project.index(project.getIndicesLookup().get(index.getName()).getWriteIndex());
         if (im.getAliases().size() > 0) {
             throw new IllegalArgumentException(
                 String.format(

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java

@@ -111,20 +111,30 @@ public class MetadataDataStreamsService {
         this.updateOptionsTaskQueue = clusterService.createTaskQueue("modify-data-stream-options", Priority.NORMAL, updateOptionsExecutor);
     }
 
-    public void modifyDataStream(final ModifyDataStreamsAction.Request request, final ActionListener<AcknowledgedResponse> listener) {
+    public void modifyDataStream(
+        final ProjectId projectId,
+        final ModifyDataStreamsAction.Request request,
+        final ActionListener<AcknowledgedResponse> listener
+    ) {
         if (request.getActions().size() == 0) {
             listener.onResponse(AcknowledgedResponse.TRUE);
         } else {
             submitUnbatchedTask("update-backing-indices", new AckedClusterStateUpdateTask(Priority.URGENT, request, listener) {
                 @Override
                 public ClusterState execute(ClusterState currentState) {
-                    return modifyDataStream(currentState, request.getActions(), indexMetadata -> {
-                        try {
-                            return indicesService.createIndexMapperServiceForValidation(indexMetadata);
-                        } catch (IOException e) {
-                            throw new IllegalStateException(e);
-                        }
-                    }, clusterService.getSettings());
+                    final var project = modifyDataStream(
+                        currentState.metadata().getProject(projectId),
+                        request.getActions(),
+                        indexMetadata -> {
+                            try {
+                                return indicesService.createIndexMapperServiceForValidation(indexMetadata);
+                            } catch (IOException e) {
+                                throw new IllegalStateException(e);
+                            }
+                        },
+                        clusterService.getSettings()
+                    );
+                    return ClusterState.builder(currentState).putProjectMetadata(project).build();
                 }
             });
         }
@@ -223,23 +233,22 @@ public class MetadataDataStreamsService {
     /**
      * Computes the resulting cluster state after applying all requested data stream modifications in order.
      *
-     * @param currentState current cluster state
-     * @param actions      ordered list of modifications to perform
+     * @param currentProject current project metadata
+     * @param actions ordered list of modifications to perform
      * @return resulting cluster state after all modifications have been performed
      */
-    static ClusterState modifyDataStream(
-        ClusterState currentState,
+    static ProjectMetadata modifyDataStream(
+        ProjectMetadata currentProject,
         Iterable<DataStreamAction> actions,
         Function<IndexMetadata, MapperService> mapperSupplier,
         Settings nodeSettings
     ) {
-        Metadata updatedMetadata = currentState.metadata();
-
+        var updatedProject = currentProject;
         for (var action : actions) {
-            Metadata.Builder builder = Metadata.builder(updatedMetadata);
+            ProjectMetadata.Builder builder = ProjectMetadata.builder(updatedProject);
             if (action.getType() == DataStreamAction.Type.ADD_BACKING_INDEX) {
                 addBackingIndex(
-                    updatedMetadata,
+                    updatedProject,
                     builder,
                     mapperSupplier,
                     action.getDataStream(),
@@ -248,14 +257,14 @@ public class MetadataDataStreamsService {
                     nodeSettings
                 );
             } else if (action.getType() == DataStreamAction.Type.REMOVE_BACKING_INDEX) {
-                removeBackingIndex(updatedMetadata, builder, action.getDataStream(), action.getIndex(), action.isFailureStore());
+                removeBackingIndex(updatedProject, builder, action.getDataStream(), action.getIndex(), action.isFailureStore());
             } else {
                 throw new IllegalStateException("unsupported data stream action type [" + action.getClass().getName() + "]");
             }
-            updatedMetadata = builder.build();
+            updatedProject = builder.build();
         }
 
-        return ClusterState.builder(currentState).metadata(updatedMetadata).build();
+        return updatedProject;
     }
 
     /**
@@ -328,21 +337,21 @@ public class MetadataDataStreamsService {
     }
 
     private static void addBackingIndex(
-        Metadata metadata,
-        Metadata.Builder builder,
+        ProjectMetadata project,
+        ProjectMetadata.Builder builder,
         Function<IndexMetadata, MapperService> mapperSupplier,
         String dataStreamName,
         String indexName,
         boolean failureStore,
         Settings nodeSettings
     ) {
-        var dataStream = validateDataStream(metadata.getProject(), dataStreamName);
-        var index = validateIndex(metadata, indexName);
+        var dataStream = validateDataStream(project, dataStreamName);
+        var index = validateIndex(project, indexName);
 
         try {
             MetadataMigrateToDataStreamService.prepareBackingIndex(
                 builder,
-                metadata.getProject().index(index.getWriteIndex()),
+                project.index(index.getWriteIndex()),
                 dataStreamName,
                 mapperSupplier,
                 false,
@@ -355,21 +364,21 @@ public class MetadataDataStreamsService {
 
         // add index to data stream
         if (failureStore) {
-            builder.put(dataStream.addFailureStoreIndex(metadata, index.getWriteIndex()));
+            builder.put(dataStream.addFailureStoreIndex(project, index.getWriteIndex()));
         } else {
-            builder.put(dataStream.addBackingIndex(metadata, index.getWriteIndex()));
+            builder.put(dataStream.addBackingIndex(project, index.getWriteIndex()));
         }
     }
 
     private static void removeBackingIndex(
-        Metadata metadata,
-        Metadata.Builder builder,
+        ProjectMetadata project,
+        ProjectMetadata.Builder builder,
         String dataStreamName,
         String indexName,
         boolean failureStore
     ) {
         boolean indexNotRemoved = true;
-        DataStream dataStream = validateDataStream(metadata.getProject(), dataStreamName);
+        DataStream dataStream = validateDataStream(project, dataStreamName);
         List<Index> targetIndices = failureStore ? dataStream.getFailureIndices() : dataStream.getIndices();
         for (Index backingIndex : targetIndices) {
             if (backingIndex.getName().equals(indexName)) {
@@ -406,8 +415,8 @@ public class MetadataDataStreamsService {
         return (DataStream) dataStream;
     }
 
-    private static IndexAbstraction validateIndex(Metadata metadata, String indexName) {
-        IndexAbstraction index = metadata.getProject().getIndicesLookup().get(indexName);
+    private static IndexAbstraction validateIndex(ProjectMetadata project, String indexName) {
+        IndexAbstraction index = project.getIndicesLookup().get(indexName);
         if (index == null || index.getType() != IndexAbstraction.Type.CONCRETE_INDEX) {
             throw new IllegalArgumentException("index [" + indexName + "] not found");
         }

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java

@@ -147,18 +147,16 @@ public class MetadataMigrateToDataStreamService {
         ActionListener<Void> listener
     ) throws Exception {
         validateRequest(currentState, request);
-        IndexAbstraction.Alias alias = (IndexAbstraction.Alias) currentState.metadata()
-            .getProject()
-            .getIndicesLookup()
-            .get(request.aliasName);
+        final var project = currentState.metadata().getProject();
+        IndexAbstraction.Alias alias = (IndexAbstraction.Alias) project.getIndicesLookup().get(request.aliasName);
 
         validateBackingIndices(currentState, request.aliasName);
-        Metadata.Builder mb = Metadata.builder(currentState.metadata());
+        ProjectMetadata.Builder mb = ProjectMetadata.builder(project);
         for (Index index : alias.getIndices()) {
-            IndexMetadata im = currentState.metadata().getProject().index(index);
-            prepareBackingIndex(mb, im, request.aliasName, mapperSupplier, true);
+            IndexMetadata im = project.index(index);
+            prepareBackingIndex(mb, im, request.aliasName, mapperSupplier, true, false, Settings.EMPTY);
         }
-        currentState = ClusterState.builder(currentState).metadata(mb).build();
+        currentState = ClusterState.builder(currentState).putProjectMetadata(mb).build();
 
         Index writeIndex = alias.getWriteIndex();
 
@@ -182,7 +180,7 @@ public class MetadataMigrateToDataStreamService {
             isDslOnlyMode,
             req,
             backingIndices,
-            currentState.metadata().getProject().index(writeIndex),
+            currentState.metadata().getProject(project.id()).index(writeIndex),
             listener,
             // No need to initialize the failure store when migrating to a data stream.
             false
@@ -207,17 +205,6 @@ public class MetadataMigrateToDataStreamService {
         }
     }
 
-    // hides the index, optionally removes the alias, and adds data stream timestamp field mapper
-    static void prepareBackingIndex(
-        Metadata.Builder b,
-        IndexMetadata im,
-        String dataStreamName,
-        Function<IndexMetadata, MapperService> mapperSupplier,
-        boolean removeAlias
-    ) throws IOException {
-        prepareBackingIndex(b, im, dataStreamName, mapperSupplier, removeAlias, false, Settings.EMPTY);
-    }
-
     /**
      * Hides the index, optionally removes the alias, adds data stream timestamp field mapper, and configures any additional settings
      * needed for the index to be included within a data stream.
@@ -232,7 +219,7 @@ public class MetadataMigrateToDataStreamService {
      * @param nodeSettings The settings for the current node
      */
     static void prepareBackingIndex(
-        Metadata.Builder b,
+        ProjectMetadata.Builder b,
         IndexMetadata im,
         String dataStreamName,
         Function<IndexMetadata, MapperService> mapperSupplier,

server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java

@@ -423,7 +423,7 @@ public class DataStreamTests extends AbstractXContentSerializingTestCase<DataStr
             false
         );
 
-        DataStream updated = original.addBackingIndex(builder.build(), indexToAdd);
+        DataStream updated = original.addBackingIndex(builder.build().getProject(), indexToAdd);
         assertThat(updated.getName(), equalTo(original.getName()));
         assertThat(updated.getGeneration(), equalTo(original.getGeneration() + 1));
         assertThat(updated.getIndices().size(), equalTo(original.getIndices().size() + 1));
@@ -449,7 +449,10 @@ public class DataStreamTests extends AbstractXContentSerializingTestCase<DataStr
 
         Index indexToAdd = randomFrom(ds2.getIndices().toArray(Index.EMPTY_ARRAY));
 
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> ds1.addBackingIndex(builder.build(), indexToAdd));
+        IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> ds1.addBackingIndex(builder.build().getProject(), indexToAdd)
+        );
         assertThat(
             e.getMessage(),
             equalTo(
@@ -479,7 +482,10 @@ public class DataStreamTests extends AbstractXContentSerializingTestCase<DataStr
 
         Index indexToAdd = randomFrom(ds2.getFailureIndices().toArray(Index.EMPTY_ARRAY));
 
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> ds1.addBackingIndex(builder.build(), indexToAdd));
+        IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> ds1.addBackingIndex(builder.build().getProject(), indexToAdd)
+        );
         assertThat(
             e.getMessage(),
             equalTo(
@@ -504,7 +510,7 @@ public class DataStreamTests extends AbstractXContentSerializingTestCase<DataStr
 
         Index indexToAdd = randomFrom(original.getIndices().toArray(Index.EMPTY_ARRAY));
 
-        DataStream updated = original.addBackingIndex(builder.build(), indexToAdd);
+        DataStream updated = original.addBackingIndex(builder.build().getProject(), indexToAdd);
         assertThat(updated.getName(), equalTo(original.getName()));
         assertThat(updated.getGeneration(), equalTo(original.getGeneration()));
         assertThat(updated.getIndices().size(), equalTo(original.getIndices().size()));
@@ -537,7 +543,7 @@ public class DataStreamTests extends AbstractXContentSerializingTestCase<DataStr
 
         IllegalArgumentException e = expectThrows(
             IllegalArgumentException.class,
-            () -> original.addBackingIndex(builder.build(), indexToAdd)
+            () -> original.addBackingIndex(builder.build().getProject(), indexToAdd)
         );
         assertThat(
             e.getMessage(),
@@ -574,7 +580,7 @@ public class DataStreamTests extends AbstractXContentSerializingTestCase<DataStr
             false
         );
 
-        DataStream updated = original.addFailureStoreIndex(builder.build(), indexToAdd);
+        DataStream updated = original.addFailureStoreIndex(builder.build().getProject(), indexToAdd);
         assertThat(updated.getName(), equalTo(original.getName()));
         assertThat(updated.getGeneration(), equalTo(original.getGeneration() + 1));
         assertThat(updated.getIndices().size(), equalTo(original.getIndices().size()));
@@ -602,7 +608,7 @@ public class DataStreamTests extends AbstractXContentSerializingTestCase<DataStr
 
         IllegalArgumentException e = expectThrows(
             IllegalArgumentException.class,
-            () -> ds1.addFailureStoreIndex(builder.build(), indexToAdd)
+            () -> ds1.addFailureStoreIndex(builder.build().getProject(), indexToAdd)
         );
         assertThat(
             e.getMessage(),
@@ -635,7 +641,7 @@ public class DataStreamTests extends AbstractXContentSerializingTestCase<DataStr
 
         IllegalArgumentException e = expectThrows(
             IllegalArgumentException.class,
-            () -> ds1.addFailureStoreIndex(builder.build(), indexToAdd)
+            () -> ds1.addFailureStoreIndex(builder.build().getProject(), indexToAdd)
         );
         assertThat(
             e.getMessage(),
@@ -662,7 +668,7 @@ public class DataStreamTests extends AbstractXContentSerializingTestCase<DataStr
 
         Index indexToAdd = randomFrom(original.getFailureIndices().toArray(Index.EMPTY_ARRAY));
 
-        DataStream updated = original.addFailureStoreIndex(builder.build(), indexToAdd);
+        DataStream updated = original.addFailureStoreIndex(builder.build().getProject(), indexToAdd);
         assertThat(updated.getName(), equalTo(original.getName()));
         assertThat(updated.getGeneration(), equalTo(original.getGeneration()));
         assertThat(updated.getIndices().size(), equalTo(original.getIndices().size()));
@@ -695,7 +701,7 @@ public class DataStreamTests extends AbstractXContentSerializingTestCase<DataStr
 
         IllegalArgumentException e = expectThrows(
             IllegalArgumentException.class,
-            () -> original.addFailureStoreIndex(builder.build(), indexToAdd)
+            () -> original.addFailureStoreIndex(builder.build().getProject(), indexToAdd)
         );
         assertThat(
             e.getMessage(),

server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java

@@ -9,7 +9,6 @@
 
 package org.elasticsearch.cluster.metadata;
 
-import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
@@ -43,7 +42,7 @@ public class MetadataDataStreamsServiceTests extends MapperServiceTestCase {
         final int numBackingIndices = randomIntBetween(1, 4);
         final String dataStreamName = randomAlphaOfLength(5);
         IndexMetadata[] backingIndices = new IndexMetadata[numBackingIndices];
-        Metadata.Builder mb = Metadata.builder();
+        ProjectMetadata.Builder mb = ProjectMetadata.builder(randomProjectIdOrDefault());
         for (int k = 0; k < numBackingIndices; k++) {
             backingIndices[k] = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, k + 1, epochMillis))
                 .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()))
@@ -64,15 +63,15 @@ public class MetadataDataStreamsServiceTests extends MapperServiceTestCase {
             .build();
         mb.put(indexToAdd, false);
 
-        ClusterState originalState = ClusterState.builder(new ClusterName("dummy")).metadata(mb.build()).build();
-        ClusterState newState = MetadataDataStreamsService.modifyDataStream(
-            originalState,
+        ProjectMetadata originalProject = mb.build();
+        ProjectMetadata newProject = MetadataDataStreamsService.modifyDataStream(
+            originalProject,
             List.of(DataStreamAction.addBackingIndex(dataStreamName, indexToAdd.getIndex().getName())),
             this::getMapperService,
             Settings.EMPTY
         );
 
-        IndexAbstraction ds = newState.metadata().getProject().getIndicesLookup().get(dataStreamName);
+        IndexAbstraction ds = newProject.getIndicesLookup().get(dataStreamName);
         assertThat(ds, notNullValue());
         assertThat(ds.getType(), equalTo(IndexAbstraction.Type.DATA_STREAM));
         assertThat(ds.getIndices().size(), equalTo(numBackingIndices + 1));
@@ -83,7 +82,7 @@ public class MetadataDataStreamsServiceTests extends MapperServiceTestCase {
                 Arrays.stream(backingIndices).map(IndexMetadata::getIndex).map(Index::getName).toList().toArray(Strings.EMPTY_ARRAY)
             )
         );
-        IndexMetadata zeroIndex = newState.metadata().getProject().index(ds.getIndices().get(0));
+        IndexMetadata zeroIndex = newProject.index(ds.getIndices().get(0));
         assertThat(zeroIndex.getIndex(), equalTo(indexToAdd.getIndex()));
         assertThat(zeroIndex.getSettings().get("index.hidden"), equalTo("true"));
         assertThat(zeroIndex.getAliases().size(), equalTo(0));
@@ -94,7 +93,7 @@ public class MetadataDataStreamsServiceTests extends MapperServiceTestCase {
         final int numBackingIndices = randomIntBetween(2, 4);
         final String dataStreamName = randomAlphaOfLength(5);
         IndexMetadata[] backingIndices = new IndexMetadata[numBackingIndices];
-        Metadata.Builder mb = Metadata.builder();
+        ProjectMetadata.Builder mb = ProjectMetadata.builder(randomProjectIdOrDefault());
         for (int k = 0; k < numBackingIndices; k++) {
             backingIndices[k] = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, k + 1, epochMillis))
                 .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()))
@@ -108,15 +107,15 @@ public class MetadataDataStreamsServiceTests extends MapperServiceTestCase {
         mb.put(DataStreamTestHelper.newInstance(dataStreamName, Arrays.stream(backingIndices).map(IndexMetadata::getIndex).toList()));
 
         final IndexMetadata indexToRemove = backingIndices[randomIntBetween(0, numBackingIndices - 2)];
-        ClusterState originalState = ClusterState.builder(new ClusterName("dummy")).metadata(mb.build()).build();
-        ClusterState newState = MetadataDataStreamsService.modifyDataStream(
-            originalState,
+        ProjectMetadata originalProject = mb.build();
+        ProjectMetadata newProject = MetadataDataStreamsService.modifyDataStream(
+            originalProject,
             List.of(DataStreamAction.removeBackingIndex(dataStreamName, indexToRemove.getIndex().getName())),
             this::getMapperService,
             Settings.EMPTY
         );
 
-        IndexAbstraction ds = newState.metadata().getProject().getIndicesLookup().get(dataStreamName);
+        IndexAbstraction ds = newProject.getIndicesLookup().get(dataStreamName);
         assertThat(ds, notNullValue());
         assertThat(ds.getType(), equalTo(IndexAbstraction.Type.DATA_STREAM));
         assertThat(ds.getIndices().size(), equalTo(numBackingIndices - 1));
@@ -127,10 +126,10 @@ public class MetadataDataStreamsServiceTests extends MapperServiceTestCase {
             .toList();
         assertThat(expectedBackingIndices, containsInAnyOrder(ds.getIndices().toArray()));
 
-        IndexMetadata removedIndex = newState.metadata().getProject().indices().get(indexToRemove.getIndex().getName());
+        IndexMetadata removedIndex = newProject.indices().get(indexToRemove.getIndex().getName());
         assertThat(removedIndex, notNullValue());
         assertThat(removedIndex.getSettings().get("index.hidden"), equalTo("false"));
-        assertNull(newState.metadata().getProject().getIndicesLookup().get(indexToRemove.getIndex().getName()).getParentDataStream());
+        assertNull(newProject.getIndicesLookup().get(indexToRemove.getIndex().getName()).getParentDataStream());
     }
 
     public void testRemoveWriteIndexIsProhibited() {
@@ -138,7 +137,7 @@ public class MetadataDataStreamsServiceTests extends MapperServiceTestCase {
         final int numBackingIndices = randomIntBetween(1, 4);
         final String dataStreamName = randomAlphaOfLength(5);
         IndexMetadata[] backingIndices = new IndexMetadata[numBackingIndices];
-        Metadata.Builder mb = Metadata.builder();
+        ProjectMetadata.Builder mb = ProjectMetadata.builder(randomProjectIdOrDefault());
         for (int k = 0; k < numBackingIndices; k++) {
             backingIndices[k] = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, k + 1, epochMillis))
                 .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()))
@@ -152,12 +151,12 @@ public class MetadataDataStreamsServiceTests extends MapperServiceTestCase {
         mb.put(DataStreamTestHelper.newInstance(dataStreamName, Arrays.stream(backingIndices).map(IndexMetadata::getIndex).toList()));
 
         final IndexMetadata indexToRemove = backingIndices[numBackingIndices - 1];
-        ClusterState originalState = ClusterState.builder(new ClusterName("dummy")).metadata(mb.build()).build();
+        ProjectMetadata originalProject = mb.build();
 
         IllegalArgumentException e = expectThrows(
             IllegalArgumentException.class,
             () -> MetadataDataStreamsService.modifyDataStream(
-                originalState,
+                originalProject,
                 List.of(DataStreamAction.removeBackingIndex(dataStreamName, indexToRemove.getIndex().getName())),
                 this::getMapperService,
                 Settings.EMPTY
@@ -182,7 +181,7 @@ public class MetadataDataStreamsServiceTests extends MapperServiceTestCase {
         final int numBackingIndices = randomIntBetween(1, 4);
         final String dataStreamName = randomAlphaOfLength(5);
         IndexMetadata[] backingIndices = new IndexMetadata[numBackingIndices];
-        Metadata.Builder mb = Metadata.builder();
+        ProjectMetadata.Builder mb = ProjectMetadata.builder(randomProjectIdOrDefault());
         for (int k = 0; k < numBackingIndices; k++) {
             backingIndices[k] = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, k + 1, epochMillis))
                 .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()))
@@ -203,9 +202,9 @@ public class MetadataDataStreamsServiceTests extends MapperServiceTestCase {
             .build();
         mb.put(indexToAdd, false);
 
-        ClusterState originalState = ClusterState.builder(new ClusterName("dummy")).metadata(mb.build()).build();
-        ClusterState newState = MetadataDataStreamsService.modifyDataStream(
-            originalState,
+        ProjectMetadata originalProject = mb.build();
+        ProjectMetadata newProject = MetadataDataStreamsService.modifyDataStream(
+            originalProject,
             List.of(
                 DataStreamAction.addBackingIndex(dataStreamName, indexToAdd.getIndex().getName()),
                 DataStreamAction.removeBackingIndex(dataStreamName, indexToAdd.getIndex().getName()),
@@ -215,7 +214,7 @@ public class MetadataDataStreamsServiceTests extends MapperServiceTestCase {
             Settings.EMPTY
         );
 
-        IndexAbstraction ds = newState.metadata().getProject().getIndicesLookup().get(dataStreamName);
+        IndexAbstraction ds = newProject.getIndicesLookup().get(dataStreamName);
         assertThat(ds, notNullValue());
         assertThat(ds.getType(), equalTo(IndexAbstraction.Type.DATA_STREAM));
         assertThat(ds.getIndices().size(), equalTo(numBackingIndices + 1));
@@ -226,7 +225,7 @@ public class MetadataDataStreamsServiceTests extends MapperServiceTestCase {
                 Arrays.stream(backingIndices).map(IndexMetadata::getIndex).map(Index::getName).toList().toArray(Strings.EMPTY_ARRAY)
             )
         );
-        IndexMetadata zeroIndex = newState.metadata().getProject().index(ds.getIndices().get(0));
+        IndexMetadata zeroIndex = newProject.index(ds.getIndices().get(0));
         assertThat(zeroIndex.getIndex(), equalTo(indexToAdd.getIndex()));
         assertThat(zeroIndex.getSettings().get("index.hidden"), equalTo("true"));
         assertThat(zeroIndex.getAliases().size(), equalTo(0));
@@ -237,7 +236,7 @@ public class MetadataDataStreamsServiceTests extends MapperServiceTestCase {
         final int numBackingIndices = randomIntBetween(1, 4);
         final String dataStreamName = randomAlphaOfLength(5);
         IndexMetadata[] backingIndices = new IndexMetadata[numBackingIndices];
-        Metadata.Builder mb = Metadata.builder();
+        ProjectMetadata.Builder mb = ProjectMetadata.builder(randomProjectIdOrDefault());
         for (int k = 0; k < numBackingIndices; k++) {
             backingIndices[k] = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, k + 1, epochMillis))
                 .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()))
@@ -258,27 +257,27 @@ public class MetadataDataStreamsServiceTests extends MapperServiceTestCase {
             .build();
         mb.put(indexToAdd, false);
 
-        ClusterState originalState = ClusterState.builder(new ClusterName("dummy")).metadata(mb.build()).build();
-        ClusterState newState = MetadataDataStreamsService.modifyDataStream(
-            originalState,
+        ProjectMetadata originalProject = mb.build();
+        ProjectMetadata newProject = MetadataDataStreamsService.modifyDataStream(
+            originalProject,
             List.of(DataStreamAction.addBackingIndex(dataStreamName, indexToAdd.getIndex().getName())),
             this::getMapperService,
             Settings.EMPTY
         );
-        newState = MetadataDataStreamsService.modifyDataStream(
-            newState,
+        newProject = MetadataDataStreamsService.modifyDataStream(
+            newProject,
             List.of(DataStreamAction.removeBackingIndex(dataStreamName, indexToAdd.getIndex().getName())),
             this::getMapperService,
             Settings.EMPTY
         );
-        newState = MetadataDataStreamsService.modifyDataStream(
-            newState,
+        newProject = MetadataDataStreamsService.modifyDataStream(
+            newProject,
             List.of(DataStreamAction.addBackingIndex(dataStreamName, indexToAdd.getIndex().getName())),
             this::getMapperService,
             Settings.EMPTY
         );
 
-        IndexAbstraction ds = newState.metadata().getProject().getIndicesLookup().get(dataStreamName);
+        IndexAbstraction ds = newProject.getIndicesLookup().get(dataStreamName);
         assertThat(ds, notNullValue());
         assertThat(ds.getType(), equalTo(IndexAbstraction.Type.DATA_STREAM));
         assertThat(ds.getIndices().size(), equalTo(numBackingIndices + 1));
@@ -289,14 +288,14 @@ public class MetadataDataStreamsServiceTests extends MapperServiceTestCase {
                 Arrays.stream(backingIndices).map(IndexMetadata::getIndex).map(Index::getName).toList().toArray(Strings.EMPTY_ARRAY)
             )
         );
-        IndexMetadata zeroIndex = newState.metadata().getProject().index(ds.getIndices().get(0));
+        IndexMetadata zeroIndex = newProject.index(ds.getIndices().get(0));
         assertThat(zeroIndex.getIndex(), equalTo(indexToAdd.getIndex()));
         assertThat(zeroIndex.getSettings().get("index.hidden"), equalTo("true"));
         assertThat(zeroIndex.getAliases().size(), equalTo(0));
     }
 
     public void testMissingDataStream() {
-        Metadata.Builder mb = Metadata.builder();
+        ProjectMetadata.Builder mb = ProjectMetadata.builder(randomProjectIdOrDefault());
         final IndexMetadata indexToAdd = IndexMetadata.builder(randomAlphaOfLength(5))
             .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()))
             .numberOfShards(1)
@@ -306,12 +305,12 @@ public class MetadataDataStreamsServiceTests extends MapperServiceTestCase {
         mb.put(indexToAdd, false);
         final String missingDataStream = randomAlphaOfLength(5);
 
-        ClusterState originalState = ClusterState.builder(new ClusterName("dummy")).metadata(mb.build()).build();
+        ProjectMetadata originalProject = mb.build();
 
         IllegalArgumentException e = expectThrows(
             IllegalArgumentException.class,
             () -> MetadataDataStreamsService.modifyDataStream(
-                originalState,
+                originalProject,
                 List.of(DataStreamAction.addBackingIndex(missingDataStream, indexToAdd.getIndex().getName())),
                 this::getMapperService,
                 Settings.EMPTY
@@ -326,7 +325,7 @@ public class MetadataDataStreamsServiceTests extends MapperServiceTestCase {
         final int numBackingIndices = randomIntBetween(1, 4);
         final String dataStreamName = randomAlphaOfLength(5);
         IndexMetadata[] backingIndices = new IndexMetadata[numBackingIndices];
-        Metadata.Builder mb = Metadata.builder();
+        ProjectMetadata.Builder mb = ProjectMetadata.builder(randomProjectIdOrDefault());
         for (int k = 0; k < numBackingIndices; k++) {
             backingIndices[k] = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, k + 1, epochMillis))
                 .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()))
@@ -340,11 +339,11 @@ public class MetadataDataStreamsServiceTests extends MapperServiceTestCase {
         mb.put(DataStreamTestHelper.newInstance(dataStreamName, Arrays.stream(backingIndices).map(IndexMetadata::getIndex).toList()));
 
         final String missingIndex = randomAlphaOfLength(5);
-        ClusterState originalState = ClusterState.builder(new ClusterName("dummy")).metadata(mb.build()).build();
+        ProjectMetadata originalProject = mb.build();
         IllegalArgumentException e = expectThrows(
             IllegalArgumentException.class,
             () -> MetadataDataStreamsService.modifyDataStream(
-                originalState,
+                originalProject,
                 List.of(DataStreamAction.addBackingIndex(dataStreamName, missingIndex)),
                 this::getMapperService,
                 Settings.EMPTY
@@ -356,40 +355,44 @@ public class MetadataDataStreamsServiceTests extends MapperServiceTestCase {
 
     public void testRemoveBrokenBackingIndexReference() {
         var dataStreamName = "my-logs";
-        var state = DataStreamTestHelper.getClusterStateWithDataStreams(List.of(new Tuple<>(dataStreamName, 2)), List.of());
-        var original = state.getMetadata().getProject().dataStreams().get(dataStreamName);
-        var broken = original.copy()
+        final var projectId = randomProjectIdOrDefault();
+        var project = DataStreamTestHelper.getClusterStateWithDataStreams(projectId, List.of(new Tuple<>(dataStreamName, 2)), List.of())
+            .metadata()
+            .getProject(projectId);
+        var originalDs = project.dataStreams().get(dataStreamName);
+        var broken = originalDs.copy()
             .setBackingIndices(
-                original.getDataComponent()
+                originalDs.getDataComponent()
                     .copy()
-                    .setIndices(List.of(new Index(original.getIndices().get(0).getName(), "broken"), original.getIndices().get(1)))
+                    .setIndices(List.of(new Index(originalDs.getIndices().get(0).getName(), "broken"), originalDs.getIndices().get(1)))
                     .build()
             )
             .build();
-        var brokenState = ClusterState.builder(state).metadata(Metadata.builder(state.getMetadata()).put(broken).build()).build();
+        var brokenProject = ProjectMetadata.builder(project).put(broken).build();
 
         var result = MetadataDataStreamsService.modifyDataStream(
-            brokenState,
+            brokenProject,
             List.of(DataStreamAction.removeBackingIndex(dataStreamName, broken.getIndices().get(0).getName())),
             this::getMapperService,
             Settings.EMPTY
         );
-        assertThat(result.getMetadata().getProject().dataStreams().get(dataStreamName).getIndices(), hasSize(1));
-        assertThat(
-            result.getMetadata().getProject().dataStreams().get(dataStreamName).getIndices().get(0),
-            equalTo(original.getIndices().get(1))
-        );
+        assertThat(result.dataStreams().get(dataStreamName).getIndices(), hasSize(1));
+        assertThat(result.dataStreams().get(dataStreamName).getIndices().get(0), equalTo(originalDs.getIndices().get(1)));
     }
 
     public void testRemoveBackingIndexThatDoesntExist() {
         var dataStreamName = "my-logs";
-        var state = DataStreamTestHelper.getClusterStateWithDataStreams(List.of(new Tuple<>(dataStreamName, 2)), List.of());
+        final var projectId = randomProjectIdOrDefault();
+        var project = DataStreamTestHelper.getClusterStateWithDataStreams(projectId, List.of(new Tuple<>(dataStreamName, 2)), List.of())
+            .metadata()
+            .getProject(projectId);
+        ;
 
         String indexToRemove = DataStream.getDefaultBackingIndexName(dataStreamName, 3);
         var e = expectThrows(
             IllegalArgumentException.class,
             () -> MetadataDataStreamsService.modifyDataStream(
-                state,
+                project,
                 List.of(DataStreamAction.removeBackingIndex(dataStreamName, indexToRemove)),
                 this::getMapperService,
                 Settings.EMPTY

server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java

@@ -445,7 +445,7 @@ public class MetadataMigrateToDataStreamServiceTests extends MapperServiceTestCa
              * Here the input indexMetadata will have the index.hidden setting set to true. So we expect no change to the settings, and
              * for the settings version to remain the same
              */
-            Metadata.Builder metadataBuilder = Metadata.builder();
+            ProjectMetadata.Builder metadataBuilder = ProjectMetadata.builder(randomProjectIdOrDefault());
             Settings indexMetadataSettings = Settings.builder()
                 .put(IndexMetadata.SETTING_INDEX_HIDDEN, true)
                 .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
@@ -465,16 +465,16 @@ public class MetadataMigrateToDataStreamServiceTests extends MapperServiceTestCa
                 failureStore,
                 nodeSettings
             );
-            Metadata metadata = metadataBuilder.build();
-            assertThat(indexMetadata.getSettings(), equalTo(metadata.getProject().index(indexName).getSettings()));
-            assertThat(metadata.getProject().index(indexName).getSettingsVersion(), equalTo(indexMetadata.getSettingsVersion()));
+            ProjectMetadata metadata = metadataBuilder.build();
+            assertThat(indexMetadata.getSettings(), equalTo(metadata.index(indexName).getSettings()));
+            assertThat(metadata.index(indexName).getSettingsVersion(), equalTo(indexMetadata.getSettingsVersion()));
         }
         {
             /*
              * Here the input indexMetadata will not have the index.hidden setting set to true. So prepareBackingIndex will add that,
              * meaning that the settings and settings version will change.
              */
-            Metadata.Builder metadataBuilder = Metadata.builder();
+            ProjectMetadata.Builder metadataBuilder = ProjectMetadata.builder(randomProjectIdOrDefault());
             Settings indexMetadataSettings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build();
             IndexMetadata indexMetadata = IndexMetadata.builder(indexName)
                 .settings(indexMetadataSettings)
@@ -491,9 +491,9 @@ public class MetadataMigrateToDataStreamServiceTests extends MapperServiceTestCa
                 failureStore,
                 nodeSettings
             );
-            Metadata metadata = metadataBuilder.build();
-            assertThat(indexMetadata.getSettings(), not(equalTo(metadata.getProject().index(indexName).getSettings())));
-            assertThat(metadata.getProject().index(indexName).getSettingsVersion(), equalTo(indexMetadata.getSettingsVersion() + 1));
+            ProjectMetadata metadata = metadataBuilder.build();
+            assertThat(indexMetadata.getSettings(), not(equalTo(metadata.index(indexName).getSettings())));
+            assertThat(metadata.index(indexName).getSettingsVersion(), equalTo(indexMetadata.getSettingsVersion() + 1));
         }
     }
 

x-pack/qa/multi-project/core-rest-tests-with-multiple-projects/build.gradle

@@ -43,7 +43,6 @@ tasks.named("yamlRestTest").configure {
     '^data_stream/10_basic/Get data stream and check DSL and ILM information',
     '^data_stream/40_supported_apis/Verify shard stores api', // uses _shard_stores API
     '^data_stream/130_migrate_to_data_stream/*',
-    '^data_stream/170_modify_data_stream/*',
     '^health/10_basic/*',
     '^health/40_diagnosis/*',
     '^indices.get_alias/10_basic/Get alias against closed indices', // Does NOT work with security enabled, see also core-rest-tests-with-security