Make ILM CRUD APIs (properly) project aware (MP-1802)

The PUT and DELETE APIs were already project aware, but pushed the
project resolver into the cluster state update task. This commit pushes
the project id down instead - which is consistent with other actions
that have been made project aware.

Additionally, this commit makes the GET ILM policy project aware and it
includes ILM's YAML REST tests in the multi-project test suite.

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java

@@ -9,10 +9,10 @@ package org.elasticsearch.xpack.core.ilm;
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.ItemUsage;
 import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
+import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.compress.NotXContentException;
 import org.elasticsearch.common.settings.Settings;
@@ -112,12 +112,10 @@ public class LifecyclePolicyUtils {
      */
     public static ItemUsage calculateUsage(
         final IndexNameExpressionResolver indexNameExpressionResolver,
-        final ClusterState state,
+        final ProjectMetadata project,
         final String policyName
     ) {
-        final List<String> indices = state.metadata()
-            .getProject()
-            .indices()
+        final List<String> indices = project.indices()
             .values()
             .stream()
             .filter(indexMetadata -> policyName.equals(indexMetadata.getLifecyclePolicyName()))
@@ -125,22 +123,22 @@ public class LifecyclePolicyUtils {
             .collect(Collectors.toList());
 
         final List<String> allDataStreams = indexNameExpressionResolver.dataStreamNames(
-            state,
+            project,
             IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN
         );
 
         final List<String> dataStreams = allDataStreams.stream().filter(dsName -> {
-            String indexTemplate = MetadataIndexTemplateService.findV2Template(state.metadata().getProject(), dsName, false);
+            String indexTemplate = MetadataIndexTemplateService.findV2Template(project, dsName, false);
             if (indexTemplate != null) {
-                Settings settings = MetadataIndexTemplateService.resolveSettings(state.metadata().getProject(), indexTemplate);
+                Settings settings = MetadataIndexTemplateService.resolveSettings(project, indexTemplate);
                 return policyName.equals(LifecycleSettings.LIFECYCLE_NAME_SETTING.get(settings));
             } else {
                 return false;
             }
         }).collect(Collectors.toList());
 
-        final List<String> composableTemplates = state.metadata().getProject().templatesV2().keySet().stream().filter(templateName -> {
-            Settings settings = MetadataIndexTemplateService.resolveSettings(state.metadata().getProject(), templateName);
+        final List<String> composableTemplates = project.templatesV2().keySet().stream().filter(templateName -> {
+            Settings settings = MetadataIndexTemplateService.resolveSettings(project, templateName);
             return policyName.equals(LifecycleSettings.LIFECYCLE_NAME_SETTING.get(settings));
         }).collect(Collectors.toList());
 

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtilsTests.java

@@ -7,15 +7,13 @@
 
 package org.elasticsearch.xpack.core.ilm;
 
-import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplateMetadata;
 import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.ItemUsage;
-import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.cluster.metadata.Template;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexVersion;
@@ -33,105 +31,89 @@ public class LifecyclePolicyUtilsTests extends ESTestCase {
 
         {
             // Test where policy does not exist
-            ClusterState state = ClusterState.builder(new ClusterName("mycluster")).build();
+            var project = ProjectMetadata.builder(randomProjectId()).build();
             assertThat(
-                LifecyclePolicyUtils.calculateUsage(iner, state, "mypolicy"),
+                LifecyclePolicyUtils.calculateUsage(iner, project, "mypolicy"),
                 equalTo(new ItemUsage(Collections.emptyList(), Collections.emptyList(), Collections.emptyList()))
             );
         }
 
         {
             // Test where policy is not used by anything
-            ClusterState state = ClusterState.builder(new ClusterName("mycluster"))
-                .metadata(
-                    Metadata.builder()
-                        .putCustom(
-                            IndexLifecycleMetadata.TYPE,
-                            new IndexLifecycleMetadata(
-                                Collections.singletonMap("mypolicy", LifecyclePolicyMetadataTests.createRandomPolicyMetadata("mypolicy")),
-                                OperationMode.RUNNING
-                            )
-                        )
-                        .build()
+            var project = ProjectMetadata.builder(randomProjectId())
+                .putCustom(
+                    IndexLifecycleMetadata.TYPE,
+                    new IndexLifecycleMetadata(
+                        Collections.singletonMap("mypolicy", LifecyclePolicyMetadataTests.createRandomPolicyMetadata("mypolicy")),
+                        OperationMode.RUNNING
+                    )
                 )
                 .build();
             assertThat(
-                LifecyclePolicyUtils.calculateUsage(iner, state, "mypolicy"),
+                LifecyclePolicyUtils.calculateUsage(iner, project, "mypolicy"),
                 equalTo(new ItemUsage(Collections.emptyList(), Collections.emptyList(), Collections.emptyList()))
             );
         }
 
         {
             // Test where policy exists and is used by an index
-            ClusterState state = ClusterState.builder(new ClusterName("mycluster"))
-                .metadata(
-                    Metadata.builder()
-                        .putCustom(
-                            IndexLifecycleMetadata.TYPE,
-                            new IndexLifecycleMetadata(
-                                Collections.singletonMap("mypolicy", LifecyclePolicyMetadataTests.createRandomPolicyMetadata("mypolicy")),
-                                OperationMode.RUNNING
-                            )
-                        )
-                        .put(
-                            IndexMetadata.builder("myindex")
-                                .settings(indexSettings(IndexVersion.current(), 1, 0).put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy"))
-                        )
-                        .build()
+            var project = ProjectMetadata.builder(randomProjectId())
+                .putCustom(
+                    IndexLifecycleMetadata.TYPE,
+                    new IndexLifecycleMetadata(
+                        Collections.singletonMap("mypolicy", LifecyclePolicyMetadataTests.createRandomPolicyMetadata("mypolicy")),
+                        OperationMode.RUNNING
+                    )
+                )
+                .put(
+                    IndexMetadata.builder("myindex")
+                        .settings(indexSettings(IndexVersion.current(), 1, 0).put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy"))
                 )
                 .build();
             assertThat(
-                LifecyclePolicyUtils.calculateUsage(iner, state, "mypolicy"),
+                LifecyclePolicyUtils.calculateUsage(iner, project, "mypolicy"),
                 equalTo(new ItemUsage(Collections.singleton("myindex"), Collections.emptyList(), Collections.emptyList()))
             );
         }
 
         {
             // Test where policy exists and is used by an index, and template
-            ClusterState state = ClusterState.builder(new ClusterName("mycluster"))
-                .metadata(
-                    Metadata.builder()
-                        .putCustom(
-                            IndexLifecycleMetadata.TYPE,
-                            new IndexLifecycleMetadata(
-                                Collections.singletonMap("mypolicy", LifecyclePolicyMetadataTests.createRandomPolicyMetadata("mypolicy")),
-                                OperationMode.RUNNING
-                            )
-                        )
-                        .put(
-                            IndexMetadata.builder("myindex")
-                                .settings(indexSettings(IndexVersion.current(), 1, 0).put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy"))
-                        )
-                        .putCustom(
-                            ComposableIndexTemplateMetadata.TYPE,
-                            new ComposableIndexTemplateMetadata(
-                                Collections.singletonMap(
-                                    "mytemplate",
-                                    ComposableIndexTemplate.builder()
-                                        .indexPatterns(Collections.singletonList("myds"))
-                                        .template(
-                                            new Template(
-                                                Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy").build(),
-                                                null,
-                                                null
-                                            )
-                                        )
-                                        .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false))
-                                        .build()
+            var project = ProjectMetadata.builder(randomProjectId())
+                .putCustom(
+                    IndexLifecycleMetadata.TYPE,
+                    new IndexLifecycleMetadata(
+                        Collections.singletonMap("mypolicy", LifecyclePolicyMetadataTests.createRandomPolicyMetadata("mypolicy")),
+                        OperationMode.RUNNING
+                    )
+                )
+                .put(
+                    IndexMetadata.builder("myindex")
+                        .settings(indexSettings(IndexVersion.current(), 1, 0).put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy"))
+                )
+                .putCustom(
+                    ComposableIndexTemplateMetadata.TYPE,
+                    new ComposableIndexTemplateMetadata(
+                        Collections.singletonMap(
+                            "mytemplate",
+                            ComposableIndexTemplate.builder()
+                                .indexPatterns(Collections.singletonList("myds"))
+                                .template(
+                                    new Template(Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy").build(), null, null)
                                 )
-                            )
+                                .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false))
+                                .build()
                         )
-                        .build()
+                    )
                 )
                 .build();
             assertThat(
-                LifecyclePolicyUtils.calculateUsage(iner, state, "mypolicy"),
+                LifecyclePolicyUtils.calculateUsage(iner, project, "mypolicy"),
                 equalTo(new ItemUsage(Collections.singleton("myindex"), Collections.emptyList(), Collections.singleton("mytemplate")))
             );
         }
 
         {
-            Metadata.Builder mBuilder = Metadata.builder()
+            var projectBuilder = ProjectMetadata.builder(randomProjectId())
                 .putCustom(
                     IndexLifecycleMetadata.TYPE,
                     new IndexLifecycleMetadata(
@@ -168,12 +150,13 @@ public class LifecyclePolicyUtilsTests extends ESTestCase {
                     )
                 );
             // Need to get the real Index instance of myindex:
-            mBuilder.put(DataStreamTestHelper.newInstance("myds", Collections.singletonList(mBuilder.get("myindex").getIndex())));
+            projectBuilder.put(
+                DataStreamTestHelper.newInstance("myds", Collections.singletonList(projectBuilder.get("myindex").getIndex()))
+            );
 
             // Test where policy exists and is used by an index, datastream, and template
-            ClusterState state = ClusterState.builder(new ClusterName("mycluster")).metadata(mBuilder.build()).build();
             assertThat(
-                LifecyclePolicyUtils.calculateUsage(iner, state, "mypolicy"),
+                LifecyclePolicyUtils.calculateUsage(iner, projectBuilder.build(), "mypolicy"),
                 equalTo(
                     new ItemUsage(Arrays.asList("myindex", "another"), Collections.singleton("myds"), Collections.singleton("mytemplate"))
                 )

x-pack/plugin/ilm/qa/rest/build.gradle

@@ -18,3 +18,15 @@ testClusters.configureEach {
   setting 'xpack.license.self_generated.type', 'trial'
   setting 'xpack.security.autoconfiguration.enabled', 'false'
 }
+
+configurations {
+  basicRestSpecs {
+    attributes {
+      attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE)
+    }
+  }
+}
+
+artifacts {
+  basicRestSpecs(new File(projectDir, "src/yamlRestTest/resources/rest-api-spec/test"))
+}

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleService.java

@@ -17,6 +17,7 @@ import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.LifecycleExecutionState;
+import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -41,7 +42,6 @@ import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.ilm.CheckShrinkReadyStep;
 import org.elasticsearch.xpack.core.ilm.DownsampleStep;
 import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata;
-import org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata;
 import org.elasticsearch.xpack.core.ilm.LifecyclePolicy;
 import org.elasticsearch.xpack.core.ilm.LifecycleSettings;
 import org.elasticsearch.xpack.core.ilm.OperationMode;
@@ -67,7 +67,6 @@ import java.util.stream.Collectors;
 import static org.elasticsearch.core.Strings.format;
 import static org.elasticsearch.xpack.core.ilm.IndexLifecycleOriginationDateParser.parseIndexNameAndExtractDate;
 import static org.elasticsearch.xpack.core.ilm.IndexLifecycleOriginationDateParser.shouldParseIndexName;
-import static org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata.EMPTY;
 import static org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata.currentILMMode;
 
 /**
@@ -181,8 +180,9 @@ public class IndexLifecycleService
 
         // TODO multi-project: this probably needs a per-project iteration
         @FixForMultiProject
-        ProjectMetadata projectMetadata = clusterState.metadata().getSingleProjectWithCustom(IndexLifecycleMetadata.TYPE);
-        if (projectMetadata != null) {
+        final ProjectMetadata projectMetadata = clusterState.metadata().getProject(Metadata.DEFAULT_PROJECT_ID);
+        final IndexLifecycleMetadata currentMetadata = projectMetadata.custom(IndexLifecycleMetadata.TYPE);
+        if (currentMetadata != null) {
             OperationMode currentMode = currentILMMode(projectMetadata);
             if (OperationMode.STOPPED.equals(currentMode)) {
                 return;
@@ -352,15 +352,17 @@ public class IndexLifecycleService
         }
 
         @FixForMultiProject
-        final ProjectMetadata project = event.state().metadata().getSingleProjectWithCustom(IndexLifecycleMetadata.TYPE);
-        if (project == null) {
+        final IndexLifecycleMetadata ilmMetadata = event.state()
+            .metadata()
+            .getProject(Metadata.DEFAULT_PROJECT_ID)
+            .custom(IndexLifecycleMetadata.TYPE);
+        if (ilmMetadata == null) {
             return;
         }
-        final IndexLifecycleMetadata ilmMetadata = project.custom(IndexLifecycleMetadata.TYPE);
-        final ProjectMetadata previousProject = event.previousState().metadata().projects().get(project.id());
-        final IndexLifecycleMetadata previousIlmMetadata = previousProject == null
-            ? null
-            : previousProject.custom(IndexLifecycleMetadata.TYPE);
+        final IndexLifecycleMetadata previousIlmMetadata = event.previousState()
+            .metadata()
+            .getProject(Metadata.DEFAULT_PROJECT_ID)
+            .custom(IndexLifecycleMetadata.TYPE);
         if (event.previousState().nodes().isLocalNodeElectedMaster() == false || ilmMetadata != previousIlmMetadata) {
             policyRegistry.update(ilmMetadata);
         }
@@ -396,19 +398,18 @@ public class IndexLifecycleService
      */
     @FixForMultiProject
     void triggerPolicies(ClusterState clusterState, boolean fromClusterStateChange) {
-        ProjectMetadata projectMetadata = clusterState.metadata().getSingleProjectWithCustom(IndexLifecycleMetadata.TYPE);
+        @FixForMultiProject
+        final var projectMetadata = clusterState.metadata().getProject(Metadata.DEFAULT_PROJECT_ID);
+        IndexLifecycleMetadata currentMetadata = projectMetadata.custom(IndexLifecycleMetadata.TYPE);
 
-        if (projectMetadata == null) {
-            @FixForMultiProject
-            LifecycleOperationMetadata operationMetadata = clusterState.metadata().getSingleProjectCustom(LifecycleOperationMetadata.TYPE);
-            OperationMode currentMode = operationMetadata == null ? EMPTY.getILMOperationMode() : operationMetadata.getILMOperationMode();
+        OperationMode currentMode = currentILMMode(projectMetadata);
+        if (currentMetadata == null) {
             if (currentMode == OperationMode.STOPPING) {
                 // There are no policies and ILM is in stopping mode, so stop ILM and get out of here
                 stopILM();
             }
             return;
         }
-        OperationMode currentMode = currentILMMode(projectMetadata);
 
         if (OperationMode.STOPPED.equals(currentMode)) {
             return;
@@ -529,6 +530,8 @@ public class IndexLifecycleService
             return Collections.emptySet();
         }
 
+        // Returning a set of strings will cause weird behavior with multiple projects
+        @FixForMultiProject
         Set<String> indicesPreventingShutdown = state.metadata()
             .projects()
             .values()

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/ReservedLifecycleAction.java

@@ -10,8 +10,6 @@ package org.elasticsearch.xpack.ilm.action;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.project.DefaultProjectResolver;
-import org.elasticsearch.core.FixForMultiProject;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.reservedstate.ReservedClusterStateHandler;
 import org.elasticsearch.reservedstate.TransformState;
@@ -84,13 +82,12 @@ public class ReservedLifecycleAction implements ReservedClusterStateHandler<List
         ClusterState state = prevState.state();
 
         for (var request : requests) {
-            @FixForMultiProject
             TransportPutLifecycleAction.UpdateLifecyclePolicyTask task = new TransportPutLifecycleAction.UpdateLifecyclePolicyTask(
+                state.metadata().getProject().id(),
                 request,
                 licenseState,
                 xContentRegistry,
-                client,
-                DefaultProjectResolver.INSTANCE // TODO multi-project: the settings file should specify the project being acted upon
+                client
             );
 
             state = task.execute(state);
@@ -102,14 +99,13 @@ public class ReservedLifecycleAction implements ReservedClusterStateHandler<List
         toDelete.removeAll(entities);
 
         for (var policyToDelete : toDelete) {
-            @FixForMultiProject
             TransportDeleteLifecycleAction.DeleteLifecyclePolicyTask task = new TransportDeleteLifecycleAction.DeleteLifecyclePolicyTask(
+                state.metadata().getProject().id(),
                 new DeleteLifecycleAction.Request(
                     RESERVED_CLUSTER_STATE_HANDLER_IGNORED_TIMEOUT,
                     RESERVED_CLUSTER_STATE_HANDLER_IGNORED_TIMEOUT,
                     policyToDelete
                 ),
-                DefaultProjectResolver.INSTANCE, // TODO multi-project: the settings file should specify the project being acted upon
                 ActionListener.noop()
             );
             state = task.execute(state);

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportDeleteLifecycleAction.java

@@ -18,6 +18,7 @@ import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.cluster.project.ProjectResolver;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -50,8 +51,8 @@ public class TransportDeleteLifecycleAction extends TransportMasterNodeAction<Re
         ClusterService clusterService,
         ThreadPool threadPool,
         ActionFilters actionFilters,
-        IndexNameExpressionResolver indexNameExpressionResolver,
-        ProjectResolver projectResolver
+        ProjectResolver projectResolver,
+        IndexNameExpressionResolver indexNameExpressionResolver
     ) {
         super(
             DeleteLifecycleAction.NAME,
@@ -69,26 +70,24 @@ public class TransportDeleteLifecycleAction extends TransportMasterNodeAction<Re
 
     @Override
     protected void masterOperation(Task task, Request request, ClusterState state, ActionListener<AcknowledgedResponse> listener) {
-        submitUnbatchedTask(
-            "delete-lifecycle-" + request.getPolicyName(),
-            new DeleteLifecyclePolicyTask(request, projectResolver, listener)
-        );
+        final var projectId = projectResolver.getProjectId();
+        submitUnbatchedTask("delete-lifecycle-" + request.getPolicyName(), new DeleteLifecyclePolicyTask(projectId, request, listener));
     }
 
     public static class DeleteLifecyclePolicyTask extends AckedClusterStateUpdateTask {
+        private final ProjectId projectId;
         private final Request request;
-        private final ProjectResolver projectResolver;
 
-        public DeleteLifecyclePolicyTask(Request request, ProjectResolver projectResolver, ActionListener<AcknowledgedResponse> listener) {
+        public DeleteLifecyclePolicyTask(ProjectId projectId, Request request, ActionListener<AcknowledgedResponse> listener) {
             super(request, listener);
+            this.projectId = projectId;
             this.request = request;
-            this.projectResolver = projectResolver;
         }
 
         @Override
         public ClusterState execute(ClusterState currentState) {
             String policyToDelete = request.getPolicyName();
-            ProjectMetadata projectMetadata = projectResolver.getProjectMetadata(currentState);
+            ProjectMetadata projectMetadata = currentState.metadata().getProject(projectId);
             List<String> indicesUsingPolicy = projectMetadata.indices()
                 .values()
                 .stream()

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportGetLifecycleAction.java

@@ -15,6 +15,7 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.project.ProjectResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.injection.guice.Inject;
@@ -39,12 +40,15 @@ import java.util.Map;
 
 public class TransportGetLifecycleAction extends TransportMasterNodeAction<Request, Response> {
 
+    private final ProjectResolver projectResolver;
+
     @Inject
     public TransportGetLifecycleAction(
         TransportService transportService,
         ClusterService clusterService,
         ThreadPool threadPool,
         ActionFilters actionFilters,
+        ProjectResolver projectResolver,
         IndexNameExpressionResolver indexNameExpressionResolver
     ) {
         super(
@@ -58,6 +62,7 @@ public class TransportGetLifecycleAction extends TransportMasterNodeAction<Reque
             Response::new,
             threadPool.executor(ThreadPool.Names.MANAGEMENT)
         );
+        this.projectResolver = projectResolver;
     }
 
     @Override
@@ -67,8 +72,9 @@ public class TransportGetLifecycleAction extends TransportMasterNodeAction<Reque
         if (cancellableTask.notifyIfCancelled(listener)) {
             return;
         }
+        var project = projectResolver.getProjectMetadata(state);
 
-        IndexLifecycleMetadata metadata = clusterService.state().metadata().getProject().custom(IndexLifecycleMetadata.TYPE);
+        IndexLifecycleMetadata metadata = project.custom(IndexLifecycleMetadata.TYPE);
         if (metadata == null) {
             if (request.getPolicyNames().length == 0) {
                 listener.onResponse(new Response(Collections.emptyList()));
@@ -106,7 +112,7 @@ public class TransportGetLifecycleAction extends TransportMasterNodeAction<Reque
                                     policyMetadata.getPolicy(),
                                     policyMetadata.getVersion(),
                                     policyMetadata.getModifiedDateString(),
-                                    LifecyclePolicyUtils.calculateUsage(indexNameExpressionResolver, state, policyMetadata.getName())
+                                    LifecyclePolicyUtils.calculateUsage(indexNameExpressionResolver, project, policyMetadata.getName())
                                 )
                             );
                         }
@@ -123,7 +129,7 @@ public class TransportGetLifecycleAction extends TransportMasterNodeAction<Reque
                             policyMetadata.getPolicy(),
                             policyMetadata.getVersion(),
                             policyMetadata.getModifiedDateString(),
-                            LifecyclePolicyUtils.calculateUsage(indexNameExpressionResolver, state, policyMetadata.getName())
+                            LifecyclePolicyUtils.calculateUsage(indexNameExpressionResolver, project, policyMetadata.getName())
                         )
                     );
                 }

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleAction.java

@@ -17,9 +17,11 @@ import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.AckedClusterStateUpdateTask;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.ProjectState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.cluster.metadata.RepositoriesMetadata;
 import org.elasticsearch.cluster.project.ProjectResolver;
@@ -114,49 +116,47 @@ public class TransportPutLifecycleAction extends TransportMasterNodeAction<PutLi
 
         LifecyclePolicy.validatePolicyName(request.getPolicy().getName());
 
-        {
-            ProjectMetadata projectMetadata = projectResolver.getProjectMetadata(state);
-            IndexLifecycleMetadata lifecycleMetadata = projectMetadata.custom(IndexLifecycleMetadata.TYPE, IndexLifecycleMetadata.EMPTY);
-            LifecyclePolicyMetadata existingPolicy = lifecycleMetadata.getPolicyMetadatas().get(request.getPolicy().getName());
-            // Make the request a no-op if the policy and filtered headers match exactly
-            if (isNoopUpdate(existingPolicy, request.getPolicy(), filteredHeaders)) {
-                listener.onResponse(AcknowledgedResponse.TRUE);
-                return;
-            }
+        ProjectMetadata projectMetadata = projectResolver.getProjectMetadata(state);
+        IndexLifecycleMetadata lifecycleMetadata = projectMetadata.custom(IndexLifecycleMetadata.TYPE, IndexLifecycleMetadata.EMPTY);
+        LifecyclePolicyMetadata existingPolicy = lifecycleMetadata.getPolicyMetadatas().get(request.getPolicy().getName());
+        // Make the request a no-op if the policy and filtered headers match exactly
+        if (isNoopUpdate(existingPolicy, request.getPolicy(), filteredHeaders)) {
+            listener.onResponse(AcknowledgedResponse.TRUE);
+            return;
         }
 
         submitUnbatchedTask(
             "put-lifecycle-" + request.getPolicy().getName(),
-            new UpdateLifecyclePolicyTask(request, listener, licenseState, filteredHeaders, xContentRegistry, client, projectResolver)
+            new UpdateLifecyclePolicyTask(projectMetadata.id(), request, listener, licenseState, filteredHeaders, xContentRegistry, client)
         );
     }
 
     public static class UpdateLifecyclePolicyTask extends AckedClusterStateUpdateTask {
+        private final ProjectId projectId;
         private final PutLifecycleRequest request;
         private final XPackLicenseState licenseState;
         private final Map<String, String> filteredHeaders;
         private final NamedXContentRegistry xContentRegistry;
         private final Client client;
         private final boolean verboseLogging;
-        private final ProjectResolver projectResolver;
 
         public UpdateLifecyclePolicyTask(
+            ProjectId projectId,
             PutLifecycleRequest request,
             ActionListener<AcknowledgedResponse> listener,
             XPackLicenseState licenseState,
             Map<String, String> filteredHeaders,
             NamedXContentRegistry xContentRegistry,
-            Client client,
-            ProjectResolver projectResolver
+            Client client
         ) {
             super(request, listener);
+            this.projectId = projectId;
             this.request = request;
             this.licenseState = licenseState;
             this.filteredHeaders = filteredHeaders;
             this.xContentRegistry = xContentRegistry;
             this.client = client;
             this.verboseLogging = true;
-            this.projectResolver = projectResolver;
         }
 
         /**
@@ -166,37 +166,35 @@ public class TransportPutLifecycleAction extends TransportMasterNodeAction<PutLi
          * It disables verbose logging and has no filtered headers.
          */
         UpdateLifecyclePolicyTask(
+            ProjectId projectId,
             PutLifecycleRequest request,
             XPackLicenseState licenseState,
             NamedXContentRegistry xContentRegistry,
-            Client client,
-            ProjectResolver projectResolver
+            Client client
         ) {
             super(request, null);
+            this.projectId = projectId;
             this.request = request;
             this.licenseState = licenseState;
             this.filteredHeaders = Collections.emptyMap();
             this.xContentRegistry = xContentRegistry;
             this.client = client;
             this.verboseLogging = false;
-            this.projectResolver = projectResolver;
         }
 
         @Override
-        public ClusterState execute(ClusterState currentState) throws Exception {
-            ProjectMetadata currentProjectMetadata = projectResolver.getProjectMetadata(currentState);
-            final IndexLifecycleMetadata currentMetadata = currentProjectMetadata.custom(
-                IndexLifecycleMetadata.TYPE,
-                IndexLifecycleMetadata.EMPTY
-            );
+        public ClusterState execute(ClusterState clusterState) throws Exception {
+            var projectState = clusterState.projectState(projectId);
+            final IndexLifecycleMetadata currentMetadata = projectState.metadata()
+                .custom(IndexLifecycleMetadata.TYPE, IndexLifecycleMetadata.EMPTY);
             final LifecyclePolicyMetadata existingPolicyMetadata = currentMetadata.getPolicyMetadatas().get(request.getPolicy().getName());
 
             // Double-check for no-op in the state update task, in case it was changed/reset in the meantime
             if (isNoopUpdate(existingPolicyMetadata, request.getPolicy(), filteredHeaders)) {
-                return currentState;
+                return clusterState;
             }
 
-            validatePrerequisites(request.getPolicy(), currentState, currentProjectMetadata, licenseState);
+            validatePrerequisites(request.getPolicy(), projectState, licenseState);
 
             long nextVersion = (existingPolicyMetadata == null) ? 1L : existingPolicyMetadata.getVersion() + 1L;
             SortedMap<String, LifecyclePolicyMetadata> newPolicies = new TreeMap<>(currentMetadata.getPolicyMetadatas());
@@ -214,11 +212,11 @@ public class TransportPutLifecycleAction extends TransportMasterNodeAction<PutLi
                     logger.info("updating index lifecycle policy [{}]", request.getPolicy().getName());
                 }
             }
-            IndexLifecycleMetadata newMetadata = new IndexLifecycleMetadata(newPolicies, currentILMMode(currentProjectMetadata));
-            ProjectMetadata newProjectMetadata = ProjectMetadata.builder(currentProjectMetadata)
+            IndexLifecycleMetadata newMetadata = new IndexLifecycleMetadata(newPolicies, currentILMMode(projectState.metadata()));
+            ProjectMetadata newProjectMetadata = ProjectMetadata.builder(projectState.metadata())
                 .putCustom(IndexLifecycleMetadata.TYPE, newMetadata)
                 .build();
-            ClusterState nonRefreshedState = ClusterState.builder(currentState).putProjectMetadata(newProjectMetadata).build();
+            ClusterState nonRefreshedState = ClusterState.builder(clusterState).putProjectMetadata(newProjectMetadata).build();
             if (oldPolicy == null) {
                 return nonRefreshedState;
             } else {
@@ -231,7 +229,7 @@ public class TransportPutLifecycleAction extends TransportMasterNodeAction<PutLi
                         lifecyclePolicyMetadata,
                         licenseState
                     );
-                    return ClusterState.builder(currentState).putProjectMetadata(refreshedProjectMetadata).build();
+                    return ClusterState.builder(clusterState).putProjectMetadata(refreshedProjectMetadata).build();
                 } catch (Exception e) {
                     logger.warn(() -> "unable to refresh indices phase JSON for updated policy [" + oldPolicy.getName() + "]", e);
                     // Revert to the non-refreshed state
@@ -266,15 +264,9 @@ public class TransportPutLifecycleAction extends TransportMasterNodeAction<PutLi
      * repositories exist, and that any referenced SLM policies exist.
      *
      * @param policy The lifecycle policy
-     * @param state The cluster state
-     * @param projectMetadata The project metadata
+     * @param state The project state
      */
-    private static void validatePrerequisites(
-        LifecyclePolicy policy,
-        ClusterState state,
-        ProjectMetadata projectMetadata,
-        XPackLicenseState licenseState
-    ) {
+    private static void validatePrerequisites(LifecyclePolicy policy, ProjectState state, XPackLicenseState licenseState) {
         List<Phase> phasesWithSearchableSnapshotActions = policy.getPhases()
             .values()
             .stream()
@@ -295,7 +287,7 @@ public class TransportPutLifecycleAction extends TransportMasterNodeAction<PutLi
         for (Phase phase : phasesWithSearchableSnapshotActions) {
             SearchableSnapshotAction action = (SearchableSnapshotAction) phase.getActions().get(SearchableSnapshotAction.NAME);
             String repository = action.getSnapshotRepository();
-            if (RepositoriesMetadata.get(state).repository(repository) == null) {
+            if (RepositoriesMetadata.get(state.cluster()).repository(repository) == null) {
                 throw new IllegalArgumentException(
                     "no such repository ["
                         + repository
@@ -319,7 +311,8 @@ public class TransportPutLifecycleAction extends TransportMasterNodeAction<PutLi
         for (Phase phase : phasesWithWaitForSnapshotActions) {
             WaitForSnapshotAction action = (WaitForSnapshotAction) phase.getActions().get(WaitForSnapshotAction.NAME);
             String slmPolicy = action.getPolicy();
-            if (projectMetadata.custom(SnapshotLifecycleMetadata.TYPE, SnapshotLifecycleMetadata.EMPTY)
+            if (state.metadata()
+                .custom(SnapshotLifecycleMetadata.TYPE, SnapshotLifecycleMetadata.EMPTY)
                 .getSnapshotConfigurations()
                 .get(slmPolicy) == null) {
                 throw new IllegalArgumentException(

x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportDeleteLifecycleActionTests.java

@@ -29,8 +29,8 @@ public class TransportDeleteLifecycleActionTests extends ESTestCase {
             mock(ClusterService.class),
             threadPool,
             mock(ActionFilters.class),
-            mock(IndexNameExpressionResolver.class),
-            mock(ProjectResolver.class)
+            mock(ProjectResolver.class),
+            mock(IndexNameExpressionResolver.class)
         );
         assertEquals(ReservedLifecycleAction.NAME, putAction.reservedStateHandlerName().get());
 

x-pack/qa/multi-project/xpack-rest-tests-with-multiple-projects/build.gradle

@@ -8,6 +8,7 @@ dependencies {
   testImplementation project(':test:yaml-rest-runner')
   testImplementation project(':x-pack:qa:multi-project:yaml-test-framework')
   testImplementation(testArtifact(project(":x-pack:plugin:security:qa:service-account"), "javaRestTest"))
+  restXpackTestConfig project(path: ':x-pack:plugin:ilm:qa:rest', configuration: "basicRestSpecs")
 }
 
 // let the yamlRestTests see the classpath of test
@@ -36,6 +37,14 @@ tasks.named("yamlRestTest").configure {
     '^esql/*/*',
     '^graph/*/*',
     '^health/*/*',
+    '^ilm/10_basic/Test Undeletable Policy In Use',
+    '^ilm/20_move_to_step/*',
+    '^ilm/30_retry/*',
+    '^ilm/40_explain_lifecycle/*',
+    '^ilm/60_operation_mode/*',
+    '^ilm/60_remove_policy_for_index/*',
+    '^ilm/70_downsampling/*',
+    '^ilm/80_health/*',
     '^logsdb/*/*',
     '^ml/3rd_party_deployment/*',
     '^ml/bucket_correlation_agg/*',