Fix single project GetClusterState API response for multiproject (#133936)

As it was noted in #132332 we have a general principle that API output for a single project should be the same regardless of whether the cluster is non-MP or MP. (However, the reserved metadata merge logic was never properly implemented to support this)
As a result of changes in that PR, response formats diverged. This PR makes the single project GetClusterState API response for MP clusters match the non-MP format.

server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequest.java

@@ -39,6 +39,7 @@ public class ClusterStateRequest extends LocalClusterStateRequest implements Ind
     private TimeValue waitForTimeout = DEFAULT_WAIT_FOR_NODE_TIMEOUT;
     private String[] indices = Strings.EMPTY_ARRAY;
     private IndicesOptions indicesOptions = IndicesOptions.lenientExpandOpen();
+    private boolean multiproject = false;
 
     public ClusterStateRequest(TimeValue masterNodeTimeout) {
         super(masterNodeTimeout);
@@ -140,6 +141,15 @@ public class ClusterStateRequest extends LocalClusterStateRequest implements Ind
         return customs;
     }
 
+    public ClusterStateRequest multiproject(boolean multiproject) {
+        this.multiproject = multiproject;
+        return this;
+    }
+
+    public boolean multiproject() {
+        return multiproject;
+    }
+
     public TimeValue waitForTimeout() {
         return waitForTimeout;
     }
@@ -200,5 +210,4 @@ public class ClusterStateRequest extends LocalClusterStateRequest implements Ind
         stringBuilder.append("master timeout [").append(masterTimeout()).append("]]");
         return stringBuilder.toString();
     }
-
 }

server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestBuilder.java

@@ -108,4 +108,12 @@ public class ClusterStateRequestBuilder extends ActionRequestBuilder<ClusterStat
         request.waitForTimeout(waitForTimeout);
         return this;
     }
+
+    /**
+     * When set then the response will be in multi-project format
+     */
+    public ClusterStateRequestBuilder setMultiproject(boolean multiproject) {
+        request.multiproject(multiproject);
+        return this;
+    }
 }

server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java

@@ -28,6 +28,8 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.cluster.metadata.ProjectMetadata;
+import org.elasticsearch.cluster.metadata.ReservedStateHandlerMetadata;
+import org.elasticsearch.cluster.metadata.ReservedStateMetadata;
 import org.elasticsearch.cluster.project.ProjectResolver;
 import org.elasticsearch.cluster.project.ProjectStateRegistry;
 import org.elasticsearch.cluster.routing.GlobalRoutingTable;
@@ -47,7 +49,9 @@ import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.function.BiPredicate;
 import java.util.function.Predicate;
@@ -189,12 +193,12 @@ public class TransportClusterStateAction extends TransportLocalClusterStateActio
     }
 
     private ClusterStateResponse buildResponse(final ClusterStateRequest request, final ClusterState rawState) {
-        final ClusterState currentState = filterClusterState(rawState);
+        final ClusterState filteredState = filterClusterState(rawState);
 
         ThreadPool.assertCurrentThreadPool(ThreadPool.Names.MANAGEMENT); // too heavy to construct & serialize cluster state without forking
 
         if (request.blocks() == false) {
-            final var blockException = currentState.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ);
+            final var blockException = filteredState.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ);
             if (blockException != null) {
                 // There's a METADATA_READ block in place, but we aren't returning it to the caller, and yet the caller needs to know that
                 // this block exists (e.g. it's the STATE_NOT_RECOVERED_BLOCK, so the rest of the state is known to be incomplete). Thus we
@@ -203,22 +207,22 @@ public class TransportClusterStateAction extends TransportLocalClusterStateActio
             }
         }
 
-        logger.trace("Serving cluster state request using version {}", currentState.version());
-        ClusterState.Builder builder = ClusterState.builder(currentState.getClusterName());
-        builder.version(currentState.version());
-        builder.stateUUID(currentState.stateUUID());
+        logger.trace("Serving cluster state request using version {}", filteredState.version());
+        ClusterState.Builder builder = ClusterState.builder(filteredState.getClusterName());
+        builder.version(filteredState.version());
+        builder.stateUUID(filteredState.stateUUID());
 
         if (request.nodes()) {
-            builder.nodes(currentState.nodes());
-            builder.nodeIdsToCompatibilityVersions(getCompatibilityVersions(currentState));
-            builder.nodeFeatures(getClusterFeatures(currentState));
+            builder.nodes(filteredState.nodes());
+            builder.nodeIdsToCompatibilityVersions(getCompatibilityVersions(filteredState));
+            builder.nodeFeatures(getClusterFeatures(filteredState));
         }
         if (request.routingTable()) {
             if (request.indices().length > 0) {
-                final GlobalRoutingTable.Builder globalRoutingTableBuilder = GlobalRoutingTable.builder(currentState.globalRoutingTable())
+                final GlobalRoutingTable.Builder globalRoutingTableBuilder = GlobalRoutingTable.builder(filteredState.globalRoutingTable())
                     .clear();
-                for (ProjectMetadata project : currentState.metadata().projects().values()) {
-                    RoutingTable projectRouting = currentState.routingTable(project.id());
+                for (ProjectMetadata project : filteredState.metadata().projects().values()) {
+                    RoutingTable projectRouting = filteredState.routingTable(project.id());
                     RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
                     String[] indices = indexNameExpressionResolver.concreteIndexNames(project, request);
                     for (String filteredIndex : indices) {
@@ -230,18 +234,18 @@ public class TransportClusterStateAction extends TransportLocalClusterStateActio
                 }
                 builder.routingTable(globalRoutingTableBuilder.build());
             } else {
-                builder.routingTable(currentState.globalRoutingTable());
+                builder.routingTable(filteredState.globalRoutingTable());
             }
         } else {
             builder.routingTable(GlobalRoutingTable.builder().build());
         }
         if (request.blocks()) {
-            builder.blocks(currentState.blocks());
+            builder.blocks(filteredState.blocks());
         }
 
         Metadata.Builder mdBuilder = Metadata.builder();
-        mdBuilder.clusterUUID(currentState.metadata().clusterUUID());
-        mdBuilder.coordinationMetadata(currentState.coordinationMetadata());
+        mdBuilder.clusterUUID(filteredState.metadata().clusterUUID());
+        mdBuilder.coordinationMetadata(filteredState.coordinationMetadata());
 
         if (request.metadata()) {
             // filter out metadata that shouldn't be returned by the API
@@ -250,14 +254,30 @@ public class TransportClusterStateAction extends TransportLocalClusterStateActio
             if (request.indices().length > 0) {
                 // if the request specified index names, then we don't want the whole metadata, just the version and projects (which will
                 // be filtered (below) to only include the relevant indices)
-                mdBuilder.version(currentState.metadata().version());
+                mdBuilder.version(filteredState.metadata().version());
             } else {
                 // If there are no requested indices, then we want all the metadata, except for customs that aren't exposed via the API
-                mdBuilder = Metadata.builder(currentState.metadata());
+                mdBuilder = Metadata.builder(filteredState.metadata());
                 mdBuilder.removeCustomIf(notApi);
+
+                if (projectResolver.supportsMultipleProjects() && request.multiproject() == false) {
+                    ProjectStateRegistry projectStateRegistry = ProjectStateRegistry.get(filteredState);
+                    if (projectStateRegistry.size() > 1) {
+                        throw new Metadata.MultiProjectPendingException(
+                            "There are multiple projects " + projectStateRegistry.knownProjects()
+                        );
+                    }
+                    var reservedStateMetadata = new HashMap<>(filteredState.metadata().reservedStateMetadata());
+                    var singleProjectReservedStateMetadata = projectStateRegistry.reservedStateMetadata(projectResolver.getProjectId());
+                    singleProjectReservedStateMetadata.forEach(
+                        (key, value) -> reservedStateMetadata.merge(key, value, this::mergeReservedStateMetadata)
+                    );
+
+                    mdBuilder.put(reservedStateMetadata);
+                }
             }
 
-            for (ProjectMetadata project : currentState.metadata().projects().values()) {
+            for (ProjectMetadata project : filteredState.metadata().projects().values()) {
                 ProjectMetadata.Builder pBuilder;
                 if (request.indices().length > 0) {
                     // if the request specified index names, then only include the project-id and indices
@@ -289,7 +309,7 @@ public class TransportClusterStateAction extends TransportLocalClusterStateActio
                 mdBuilder.put(pBuilder);
             }
         } else {
-            for (ProjectId project : currentState.metadata().projects().keySet()) {
+            for (ProjectId project : filteredState.metadata().projects().keySet()) {
                 // Request doesn't want to retrieve metadata, so we just fill in empty projects
                 // (because we can't have a truly empty Metadata)
                 mdBuilder.put(ProjectMetadata.builder(project));
@@ -298,14 +318,45 @@ public class TransportClusterStateAction extends TransportLocalClusterStateActio
         builder.metadata(mdBuilder);
 
         if (request.customs()) {
-            for (Map.Entry<String, ClusterState.Custom> custom : currentState.customs().entrySet()) {
+            for (Map.Entry<String, ClusterState.Custom> custom : filteredState.customs().entrySet()) {
                 if (custom.getValue().isPrivate() == false) {
                     builder.putCustom(custom.getKey(), custom.getValue());
                 }
             }
         }
 
-        return new ClusterStateResponse(currentState.getClusterName(), builder.build(), false);
+        return new ClusterStateResponse(filteredState.getClusterName(), builder.build(), false);
     }
 
+    private ReservedStateMetadata mergeReservedStateMetadata(
+        ReservedStateMetadata clusterReservedMetadata,
+        ReservedStateMetadata projectReservedMetadata
+    ) {
+        if (Objects.equals(clusterReservedMetadata.version(), projectReservedMetadata.version()) == false) {
+            logger.info(
+                "Reserved state metadata version is different for Metadata ({}) and the requested project ({})",
+                clusterReservedMetadata.version(),
+                projectReservedMetadata.version()
+            );
+        }
+        ReservedStateMetadata.Builder builder = ReservedStateMetadata.builder(clusterReservedMetadata.namespace())
+            .version(Math.max(clusterReservedMetadata.version(), projectReservedMetadata.version()));
+
+        for (ReservedStateHandlerMetadata handler : clusterReservedMetadata.handlers().values()) {
+            builder.putHandler(handler);
+        }
+        for (Map.Entry<String, ReservedStateHandlerMetadata> handlerEntry : projectReservedMetadata.handlers().entrySet()) {
+            assert clusterReservedMetadata.handlers().containsKey(handlerEntry.getKey()) == false
+                : "Duplicate of handler: " + handlerEntry.getKey();
+            builder.putHandler(handlerEntry.getValue());
+        }
+
+        if (projectReservedMetadata.errorMetadata() != null) {
+            builder.errorMetadata(projectReservedMetadata.errorMetadata());
+        } else if (clusterReservedMetadata.errorMetadata() != null) {
+            builder.errorMetadata(clusterReservedMetadata.errorMetadata());
+        }
+
+        return builder.build();
+    }
 }

server/src/main/java/org/elasticsearch/cluster/project/ProjectStateRegistry.java

@@ -204,8 +204,7 @@ public class ProjectStateRegistry extends AbstractNamedDiffable<Custom> implemen
         return projectsMarkedForDeletionGeneration;
     }
 
-    // visible for testing
-    Set<ProjectId> knownProjects() {
+    public Set<ProjectId> knownProjects() {
         return projectsEntries.keySet();
     }
 

server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateAction.java

@@ -112,6 +112,7 @@ public class RestClusterStateAction extends BaseRestHandler {
         final Map<String, String> params;
         if (request.paramAsBoolean("multi_project", false)) {
             params = Map.of(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_API, "multi-project", "true");
+            clusterStateRequest.multiproject(true);
         } else {
             params = Map.of(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_API);
         }

server/src/test/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateActionTests.java

@@ -19,6 +19,8 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.cluster.metadata.ProjectMetadata;
+import org.elasticsearch.cluster.metadata.ReservedStateHandlerMetadata;
+import org.elasticsearch.cluster.metadata.ReservedStateMetadata;
 import org.elasticsearch.cluster.node.VersionInformation;
 import org.elasticsearch.cluster.project.DefaultProjectResolver;
 import org.elasticsearch.cluster.project.ProjectResolver;
@@ -94,7 +96,7 @@ public class TransportClusterStateActionTests extends ESTestCase {
         final ProjectResolver projectResolver = DefaultProjectResolver.INSTANCE;
 
         final Set<String> indexNames = randomSet(1, 8, () -> randomAlphaOfLengthBetween(4, 12));
-        final ClusterStateRequest request = buildRandomRequest(indexNames);
+        final ClusterStateRequest request = buildRandomRequest(indexNames, false);
         final String[] expectedIndices = getExpectedIndices(request, indexNames);
 
         final ProjectId projectId = Metadata.DEFAULT_PROJECT_ID;
@@ -112,7 +114,7 @@ public class TransportClusterStateActionTests extends ESTestCase {
         final ProjectId projectId = randomUniqueProjectId();
 
         final ProjectResolver projectResolver = TestProjectResolvers.singleProject(projectId);
-        final ClusterStateRequest request = buildRandomRequest(indexNames);
+        final ClusterStateRequest request = buildRandomRequest(indexNames, false);
         final String[] expectedIndices = getExpectedIndices(request, indexNames);
 
         final int numberOfProjects = randomIntBetween(2, 5);
@@ -141,7 +143,7 @@ public class TransportClusterStateActionTests extends ESTestCase {
         final ClusterState state = buildClusterState(projects);
 
         final ProjectResolver projectResolver = TestProjectResolvers.allProjects();
-        final ClusterStateRequest request = buildRandomRequest(indexNames);
+        final ClusterStateRequest request = buildRandomRequest(indexNames, true);
         final Set<String> requestedIndices = Set.of(getExpectedIndices(request, indexNames));
 
         final ClusterStateResponse response = executeAction(projectResolver, request, state);
@@ -190,6 +192,22 @@ public class TransportClusterStateActionTests extends ESTestCase {
         assertThat(metadata.projects().keySet(), contains(projectId));
         if (request.metadata()) {
             assertThat(metadata.getProject(projectId).indices().keySet(), containsInAnyOrder(expectedIndices));
+
+            if (request.indices().length == 0) {
+                Map<String, ReservedStateMetadata> reservedStateMetadataMap = metadata.reservedStateMetadata();
+                assertThat(reservedStateMetadataMap, aMapWithSize(1));
+                ReservedStateMetadata fileSettings = reservedStateMetadataMap.get("file_settings");
+                assertNotNull(fileSettings);
+                assertThat(fileSettings.version(), equalTo(43L));
+                Map<String, ReservedStateHandlerMetadata> handlers = fileSettings.handlers();
+                assertThat(handlers, aMapWithSize(2));
+                ReservedStateHandlerMetadata clusterSettingsHandler = handlers.get("cluster_settings");
+                assertNotNull(clusterSettingsHandler);
+                assertThat(clusterSettingsHandler.keys(), containsInAnyOrder("setting_1", "setting_2"));
+                ReservedStateHandlerMetadata projectSettingsHandler = handlers.get("project_settings");
+                assertNotNull(projectSettingsHandler);
+                assertThat(projectSettingsHandler.keys(), containsInAnyOrder("setting_1"));
+            }
         } else {
             assertThat(metadata.getProject(projectId).indices(), anEmptyMap());
         }
@@ -235,7 +253,7 @@ public class TransportClusterStateActionTests extends ESTestCase {
         }
     }
 
-    private static ClusterStateRequest buildRandomRequest(Set<String> indexNames) {
+    private static ClusterStateRequest buildRandomRequest(Set<String> indexNames, boolean multipleProjects) {
         final ClusterStateRequest request = new ClusterStateRequest(TEST_REQUEST_TIMEOUT);
         if (randomBoolean()) {
             final int numberSelectedIndices = randomIntBetween(1, indexNames.size());
@@ -248,18 +266,31 @@ public class TransportClusterStateActionTests extends ESTestCase {
         request.routingTable(randomBoolean());
         request.blocks(randomBoolean());
         request.customs(true);
+        request.multiproject(multipleProjects);
         return request;
     }
 
     private static ClusterState buildClusterState(ProjectMetadata.Builder... projects) {
         final Metadata.Builder metadataBuilder = Metadata.builder();
+        metadataBuilder.put(
+            ReservedStateMetadata.builder("file_settings")
+                .version(43L)
+                .putHandler(new ReservedStateHandlerMetadata("cluster_settings", Set.of("setting_1", "setting_2")))
+                .build()
+        );
         Arrays.stream(projects).forEach(metadataBuilder::put);
         final var metadata = metadataBuilder.build();
 
         ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName(randomAlphaOfLengthBetween(4, 12)));
         ProjectStateRegistry.Builder psBuilder = ProjectStateRegistry.builder();
         for (ProjectMetadata.Builder project : projects) {
-            psBuilder.putProjectSettings(project.getId(), Settings.builder().put("setting_1", randomIdentifier()).build());
+            psBuilder.putReservedStateMetadata(
+                project.getId(),
+                ReservedStateMetadata.builder("file_settings")
+                    .version(43L)
+                    .putHandler(new ReservedStateHandlerMetadata("project_settings", Set.of("setting_1")))
+                    .build()
+            ).putProjectSettings(project.getId(), Settings.builder().put("setting_1", randomIdentifier()).build());
         }
         return csBuilder.metadata(metadata)
             .routingTable(GlobalRoutingTableTestHelper.buildRoutingTable(metadata, RoutingTable.Builder::addAsNew))

test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java

@@ -33,6 +33,7 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
 import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest;
 import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse;
@@ -1396,65 +1397,63 @@ public abstract class ESIntegTestCase extends ESTestCase {
         final List<SubscribableListener<ClusterStateResponse>> localStates = new ArrayList<>(cluster().size());
         final var masterName = internalCluster().getMasterName();
         for (Client client : cluster().getClients()) {
-            localStates.add(
-                SubscribableListener.newForked(l -> client.admin().cluster().prepareState(TEST_REQUEST_TIMEOUT).all().execute(l))
-            );
+            localStates.add(SubscribableListener.newForked(l -> prepareClusterStateRequest(client).execute(l)));
         }
         try (RefCountingListener refCountingListener = new RefCountingListener(future)) {
-            SubscribableListener.<ClusterStateResponse>newForked(
-                l -> client(masterName).admin().cluster().prepareState(TEST_REQUEST_TIMEOUT).all().execute(l)
-            ).andThenAccept(masterStateResponse -> {
-                byte[] masterClusterStateBytes = ClusterState.Builder.toBytes(masterStateResponse.getState());
-                // remove local node reference
-                final ClusterState masterClusterState = ClusterState.Builder.fromBytes(
-                    masterClusterStateBytes,
-                    null,
-                    namedWriteableRegistry
-                );
-                Map<String, Object> masterStateMap = convertToMap(masterClusterState, xContentParams());
-                String masterId = masterClusterState.nodes().getMasterNodeId();
-                if (masterId == null) {
-                    logger.warn("Failed to find an elected master in the cluster state: " + masterClusterState);
-                    throw new AssertionError("Unable to find master in cluster state. Expecting a stable master node");
-                }
-                for (SubscribableListener<ClusterStateResponse> localStateListener : localStates) {
-                    localStateListener.andThenAccept(localClusterStateResponse -> {
-                        byte[] localClusterStateBytes = ClusterState.Builder.toBytes(localClusterStateResponse.getState());
-                        // remove local node reference
-                        final ClusterState localClusterState = ClusterState.Builder.fromBytes(
-                            localClusterStateBytes,
-                            null,
-                            namedWriteableRegistry
-                        );
-                        final Map<String, Object> localStateMap = convertToMap(localClusterState, xContentParams());
-                        // Check that the non-master node has the same version of the cluster state as the master and
-                        // that the master node matches the master (otherwise there is no requirement for the cluster state to
-                        // match)
-                        if (masterClusterState.version() == localClusterState.version()
-                            && masterId.equals(localClusterState.nodes().getMasterNodeId())) {
-                            try {
-                                assertEquals(
-                                    "cluster state UUID does not match",
-                                    masterClusterState.stateUUID(),
-                                    localClusterState.stateUUID()
-                                );
-                                // Compare JSON serialization
-                                assertNull(
-                                    "cluster state JSON serialization does not match",
-                                    differenceBetweenMapsIgnoringArrayOrder(masterStateMap, localStateMap)
-                                );
-                            } catch (final AssertionError error) {
-                                logger.error(
-                                    "Cluster state from master:\n{}\nLocal cluster state:\n{}",
-                                    masterClusterState.toString(),
-                                    localClusterState.toString()
-                                );
-                                throw error;
+            SubscribableListener.<ClusterStateResponse>newForked(l -> prepareClusterStateRequest(client(masterName)).execute(l))
+                .andThenAccept(masterStateResponse -> {
+                    byte[] masterClusterStateBytes = ClusterState.Builder.toBytes(masterStateResponse.getState());
+                    // remove local node reference
+                    final ClusterState masterClusterState = ClusterState.Builder.fromBytes(
+                        masterClusterStateBytes,
+                        null,
+                        namedWriteableRegistry
+                    );
+                    Map<String, Object> masterStateMap = convertToMap(masterClusterState, xContentParams());
+                    String masterId = masterClusterState.nodes().getMasterNodeId();
+                    if (masterId == null) {
+                        logger.warn("Failed to find an elected master in the cluster state: " + masterClusterState);
+                        throw new AssertionError("Unable to find master in cluster state. Expecting a stable master node");
+                    }
+                    for (SubscribableListener<ClusterStateResponse> localStateListener : localStates) {
+                        localStateListener.andThenAccept(localClusterStateResponse -> {
+                            byte[] localClusterStateBytes = ClusterState.Builder.toBytes(localClusterStateResponse.getState());
+                            // remove local node reference
+                            final ClusterState localClusterState = ClusterState.Builder.fromBytes(
+                                localClusterStateBytes,
+                                null,
+                                namedWriteableRegistry
+                            );
+                            final Map<String, Object> localStateMap = convertToMap(localClusterState, xContentParams());
+                            // Check that the non-master node has the same version of the cluster state as the master and
+                            // that the master node matches the master (otherwise there is no requirement for the cluster state to
+                            // match)
+                            if (masterClusterState.version() == localClusterState.version()
+                                && masterId.equals(localClusterState.nodes().getMasterNodeId())) {
+                                try {
+                                    assertEquals(
+                                        "cluster state UUID does not match",
+                                        masterClusterState.stateUUID(),
+                                        localClusterState.stateUUID()
+                                    );
+                                    // Compare JSON serialization
+                                    assertNull(
+                                        "cluster state JSON serialization does not match",
+                                        differenceBetweenMapsIgnoringArrayOrder(masterStateMap, localStateMap)
+                                    );
+                                } catch (final AssertionError error) {
+                                    logger.error(
+                                        "Cluster state from master:\n{}\nLocal cluster state:\n{}",
+                                        masterClusterState.toString(),
+                                        localClusterState.toString()
+                                    );
+                                    throw error;
+                                }
                             }
-                        }
-                    }).addListener(refCountingListener.acquire());
-                }
-            }).addListener(refCountingListener.acquire());
+                        }).addListener(refCountingListener.acquire());
+                    }
+                })
+                .addListener(refCountingListener.acquire());
         }
         safeGet(future);
     }
@@ -1462,7 +1461,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
     protected void ensureClusterStateCanBeReadByNodeTool() throws IOException {
         if (cluster() != null && cluster().size() > 0) {
             final Client masterClient = client();
-            Metadata metadata = masterClient.admin().cluster().prepareState(TEST_REQUEST_TIMEOUT).all().get().getState().metadata();
+            Metadata metadata = prepareClusterStateRequest(masterClient).get().getState().metadata();
             final Map<String, String> serializationParams = Maps.newMapWithExpectedSize(2);
             serializationParams.put("binary", "true");
             serializationParams.put(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_GATEWAY);
@@ -1565,6 +1564,10 @@ public abstract class ESIntegTestCase extends ESTestCase {
         }
     }
 
+    private ClusterStateRequestBuilder prepareClusterStateRequest(Client client) {
+        return client.admin().cluster().prepareState(TEST_REQUEST_TIMEOUT).all().setMultiproject(multiProjectIntegrationTest());
+    }
+
     private static void ensureClusterInfoServiceRunning() {
         if (isInternalCluster() && cluster().size() > 0) {
             // ensures that the cluster info service didn't leak its async task, which would prevent future refreshes

test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java

@@ -2065,7 +2065,14 @@ public final class InternalTestCluster extends TestCluster {
         }
         try {
             ClusterServiceUtils.awaitClusterState(state -> state.nodes().getMasterNode() != null, clusterService(viaNode));
-            final ClusterState state = client(viaNode).admin().cluster().prepareState(TEST_REQUEST_TIMEOUT).get().getState();
+            final ClusterState state = client(viaNode).admin()
+                .cluster()
+                .prepareState(TEST_REQUEST_TIMEOUT)
+                .clear()
+                .setBlocks(true)
+                .setNodes(true)
+                .get()
+                .getState();
             final DiscoveryNode masterNode = state.nodes().getMasterNode();
             if (masterNode == null) {
                 throw new AssertionError("Master is not stable but the method expects a stable master node");