Revert "Cluster state role mapper file settings service (#107886)" (#108346)

This reverts commit 391136c0899ca0a4f8874b89cd05d5ea44987497.

docs/changelog/107886.yaml

@@ -1,5 +0,0 @@
-pr: 107886
-summary: Cluster state role mapper file settings service
-area: Authorization
-type: enhancement
-issues: []

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/rolemapping/PutRoleMappingRequest.java

@@ -166,4 +166,16 @@ public class PutRoleMappingRequest extends ActionRequest implements WriteRequest
     public ExpressionRoleMapping getMapping() {
         return new ExpressionRoleMapping(name, rules, roles, roleTemplates, metadata, enabled);
     }
+
+    public static PutRoleMappingRequest fromMapping(ExpressionRoleMapping mapping) {
+        var request = new PutRoleMappingRequest();
+        request.setName(mapping.getName());
+        request.setEnabled(mapping.isEnabled());
+        request.setRoles(mapping.getRoles());
+        request.setRoleTemplates(mapping.getRoleTemplates());
+        request.setRules(mapping.getExpression());
+        request.setMetadata(mapping.getMetadata());
+
+        return request;
+    }
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/rolemapping/PutRoleMappingRequestBuilder.java

@@ -9,7 +9,8 @@ package org.elasticsearch.xpack.core.security.action.rolemapping;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.support.WriteRequestBuilder;
 import org.elasticsearch.client.internal.ElasticsearchClient;
-import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping;
 import org.elasticsearch.xpack.core.security.authc.support.mapper.TemplateRoleName;
 import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.RoleMapperExpression;
@@ -34,8 +35,8 @@ public class PutRoleMappingRequestBuilder extends ActionRequestBuilder<PutRoleMa
     /**
      * Populate the put role request from the source and the role's name
      */
-    public PutRoleMappingRequestBuilder source(String name, XContentParser parser) throws IOException {
-        ExpressionRoleMapping mapping = ExpressionRoleMapping.parse(name, parser);
+    public PutRoleMappingRequestBuilder source(String name, BytesReference source, XContentType xContentType) throws IOException {
+        ExpressionRoleMapping mapping = ExpressionRoleMapping.parse(name, source, xContentType);
         request.setName(name);
         request.setEnabled(mapping.isEnabled());
         request.setRoles(mapping.getRoles());

x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/RoleMappingFileSettingsIT.java

@@ -7,13 +7,11 @@
 
 package org.elasticsearch.integration;
 
-import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
-import org.elasticsearch.action.admin.indices.open.OpenIndexRequest;
-import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.action.admin.indices.close.CloseIndexResponse;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.metadata.ReservedStateErrorMetadata;
@@ -27,15 +25,10 @@ import org.elasticsearch.reservedstate.action.ReservedClusterSettingsAction;
 import org.elasticsearch.reservedstate.service.FileSettingsService;
 import org.elasticsearch.test.NativeRealmIntegTestCase;
 import org.elasticsearch.xcontent.XContentParserConfiguration;
-import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingAction;
-import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingRequest;
 import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsAction;
 import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsRequest;
 import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingAction;
 import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingRequest;
-import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingRequestBuilder;
-import org.elasticsearch.xpack.core.security.authc.RealmConfig;
-import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper;
 import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping;
 import org.elasticsearch.xpack.security.action.rolemapping.ReservedRoleMappingAction;
 import org.junit.After;
@@ -46,31 +39,25 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
 import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
-import java.util.function.Consumer;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.indices.recovery.RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING;
 import static org.elasticsearch.xcontent.XContentType.JSON;
 import static org.elasticsearch.xpack.core.security.test.TestRestrictedIndices.INTERNAL_SECURITY_MAIN_INDEX_7;
 import static org.hamcrest.Matchers.allOf;
-import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.empty;
-import static org.hamcrest.Matchers.emptyArray;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.notNullValue;
-import static org.mockito.Mockito.mock;
 
 /**
- * Tests that file settings service can properly add role mappings.
+ * Tests that file settings service can properly add role mappings and detect REST clashes
+ * with the reserved role mappings.
  */
 public class RoleMappingFileSettingsIT extends NativeRealmIntegTestCase {
 
@@ -148,21 +135,12 @@ public class RoleMappingFileSettingsIT extends NativeRealmIntegTestCase {
              }
         }""";
 
-    @Override
-    protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
-        Settings.Builder builder = Settings.builder()
-            .put(super.nodeSettings(nodeOrdinal, otherSettings))
-            // some tests make use of cluster-state based role mappings
-            .put("xpack.security.authc.cluster_state_role_mappings.enabled", true);
-        return builder.build();
-    }
-
     @After
     public void cleanUp() {
         updateClusterSettings(Settings.builder().putNull("indices.recovery.max_bytes_per_sec"));
     }
 
-    public static void writeJSONFile(String node, String json, Logger logger, AtomicLong versionCounter) throws Exception {
+    private void writeJSONFile(String node, String json) throws Exception {
         long version = versionCounter.incrementAndGet();
 
         FileSettingsService fileSettingsService = internalCluster().getInstance(FileSettingsService.class, node);
@@ -173,11 +151,10 @@ public class RoleMappingFileSettingsIT extends NativeRealmIntegTestCase {
         Files.createDirectories(fileSettingsService.watchedFileDir());
         Path tempFilePath = createTempFile();
 
-        logger.info("--> before writing JSON config to node {} with path {}", node, tempFilePath);
+        logger.info("--> writing JSON config to node {} with path {}", node, tempFilePath);
         logger.info(Strings.format(json, version));
         Files.write(tempFilePath, Strings.format(json, version).getBytes(StandardCharsets.UTF_8));
         Files.move(tempFilePath, fileSettingsService.watchedFile(), StandardCopyOption.ATOMIC_MOVE);
-        logger.info("--> after writing JSON config to node {} with path {}", node, tempFilePath);
     }
 
     private Tuple<CountDownLatch, AtomicLong> setupClusterStateListener(String node, String expectedKey) {
@@ -261,41 +238,49 @@ public class RoleMappingFileSettingsIT extends NativeRealmIntegTestCase {
             expectThrows(ExecutionException.class, () -> clusterAdmin().updateSettings(req).get()).getMessage()
         );
 
-        for (UserRoleMapper userRoleMapper : internalCluster().getInstances(UserRoleMapper.class)) {
-            PlainActionFuture<Set<String>> resolveRolesFuture = new PlainActionFuture<>();
-            userRoleMapper.resolveRoles(
-                new UserRoleMapper.UserData("anyUsername", null, List.of(), Map.of(), mock(RealmConfig.class)),
-                resolveRolesFuture
-            );
-            assertThat(resolveRolesFuture.get(), containsInAnyOrder("kibana_user", "fleet_user"));
-        }
-
-        // the role mappings are not retrievable by the role mapping action (which only accesses "native" i.e. index-based role mappings)
         var request = new GetRoleMappingsRequest();
         request.setNames("everyone_kibana", "everyone_fleet");
         var response = client().execute(GetRoleMappingsAction.INSTANCE, request).get();
-        assertFalse(response.hasMappings());
-        assertThat(response.mappings(), emptyArray());
+        assertTrue(response.hasMappings());
+        assertThat(
+            Arrays.stream(response.mappings()).map(r -> r.getName()).collect(Collectors.toSet()),
+            allOf(notNullValue(), containsInAnyOrder("everyone_kibana", "everyone_fleet"))
+        );
 
-        // role mappings (with the same names) can also be stored in the "native" store
-        var putRoleMappingResponse = client().execute(PutRoleMappingAction.INSTANCE, sampleRestRequest("everyone_kibana")).actionGet();
-        assertTrue(putRoleMappingResponse.isCreated());
-        putRoleMappingResponse = client().execute(PutRoleMappingAction.INSTANCE, sampleRestRequest("everyone_fleet")).actionGet();
-        assertTrue(putRoleMappingResponse.isCreated());
+        // Try using the REST API to update the everyone_kibana role mapping
+        // This should fail, we have reserved certain role mappings in operator mode
+        assertEquals(
+            "Failed to process request "
+                + "[org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingRequest/unset] "
+                + "with errors: [[everyone_kibana] set as read-only by [file_settings]]",
+            expectThrows(
+                IllegalArgumentException.class,
+                () -> client().execute(PutRoleMappingAction.INSTANCE, sampleRestRequest("everyone_kibana")).actionGet()
+            ).getMessage()
+        );
+        assertEquals(
+            "Failed to process request "
+                + "[org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingRequest/unset] "
+                + "with errors: [[everyone_fleet] set as read-only by [file_settings]]",
+            expectThrows(
+                IllegalArgumentException.class,
+                () -> client().execute(PutRoleMappingAction.INSTANCE, sampleRestRequest("everyone_fleet")).actionGet()
+            ).getMessage()
+        );
     }
 
     public void testRoleMappingsApplied() throws Exception {
         ensureGreen();
 
         var savedClusterState = setupClusterStateListener(internalCluster().getMasterName(), "everyone_kibana");
-        writeJSONFile(internalCluster().getMasterName(), testJSON, logger, versionCounter);
+        writeJSONFile(internalCluster().getMasterName(), testJSON);
 
         assertRoleMappingsSaveOK(savedClusterState.v1(), savedClusterState.v2());
         logger.info("---> cleanup cluster settings...");
 
         savedClusterState = setupClusterStateListenerForCleanup(internalCluster().getMasterName());
 
-        writeJSONFile(internalCluster().getMasterName(), emptyJSON, logger, versionCounter);
+        writeJSONFile(internalCluster().getMasterName(), emptyJSON);
         boolean awaitSuccessful = savedClusterState.v1().await(20, TimeUnit.SECONDS);
         assertTrue(awaitSuccessful);
 
@@ -307,65 +292,32 @@ public class RoleMappingFileSettingsIT extends NativeRealmIntegTestCase {
             clusterStateResponse.getState().metadata().persistentSettings().get(INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey())
         );
 
-        // native role mappings are not affected by the removal of the cluster-state based ones
-        {
-            var request = new GetRoleMappingsRequest();
-            request.setNames("everyone_kibana", "everyone_fleet");
-            var response = client().execute(GetRoleMappingsAction.INSTANCE, request).get();
-            assertTrue(response.hasMappings());
-            assertThat(
-                Arrays.stream(response.mappings()).map(ExpressionRoleMapping::getName).toList(),
-                containsInAnyOrder("everyone_kibana", "everyone_fleet")
-            );
-        }
-
-        // and roles are resolved based on the native role mappings
-        for (UserRoleMapper userRoleMapper : internalCluster().getInstances(UserRoleMapper.class)) {
-            PlainActionFuture<Set<String>> resolveRolesFuture = new PlainActionFuture<>();
-            userRoleMapper.resolveRoles(
-                new UserRoleMapper.UserData("anyUsername", null, List.of(), Map.of(), mock(RealmConfig.class)),
-                resolveRolesFuture
-            );
-            assertThat(resolveRolesFuture.get(), contains("kibana_user_native"));
-        }
-
-        {
-            var request = new DeleteRoleMappingRequest();
-            request.setName("everyone_kibana");
-            var response = client().execute(DeleteRoleMappingAction.INSTANCE, request).get();
-            assertTrue(response.isFound());
-            request = new DeleteRoleMappingRequest();
-            request.setName("everyone_fleet");
-            response = client().execute(DeleteRoleMappingAction.INSTANCE, request).get();
-            assertTrue(response.isFound());
-        }
-
-        // no roles are resolved now, because both native and cluster-state based stores have been cleared
-        for (UserRoleMapper userRoleMapper : internalCluster().getInstances(UserRoleMapper.class)) {
-            PlainActionFuture<Set<String>> resolveRolesFuture = new PlainActionFuture<>();
-            userRoleMapper.resolveRoles(
-                new UserRoleMapper.UserData("anyUsername", null, List.of(), Map.of(), mock(RealmConfig.class)),
-                resolveRolesFuture
-            );
-            assertThat(resolveRolesFuture.get(), empty());
-        }
+        var request = new GetRoleMappingsRequest();
+        request.setNames("everyone_kibana", "everyone_fleet");
+        var response = client().execute(GetRoleMappingsAction.INSTANCE, request).get();
+        assertFalse(response.hasMappings());
     }
 
-    public static Tuple<CountDownLatch, AtomicLong> setupClusterStateListenerForError(
-        ClusterService clusterService,
-        Consumer<ReservedStateErrorMetadata> errorMetadataConsumer
-    ) {
+    private Tuple<CountDownLatch, AtomicLong> setupClusterStateListenerForError(String node) {
+        ClusterService clusterService = internalCluster().clusterService(node);
         CountDownLatch savedClusterState = new CountDownLatch(1);
         AtomicLong metadataVersion = new AtomicLong(-1);
         clusterService.addListener(new ClusterStateListener() {
             @Override
             public void clusterChanged(ClusterChangedEvent event) {
                 ReservedStateMetadata reservedState = event.state().metadata().reservedStateMetadata().get(FileSettingsService.NAMESPACE);
-                if (reservedState != null && reservedState.errorMetadata() != null) {
+                if (reservedState != null
+                    && reservedState.errorMetadata() != null
+                    && reservedState.errorMetadata().errorKind() == ReservedStateErrorMetadata.ErrorKind.PARSING) {
                     clusterService.removeListener(this);
                     metadataVersion.set(event.state().metadata().version());
                     savedClusterState.countDown();
-                    errorMetadataConsumer.accept(reservedState.errorMetadata());
+                    assertEquals(ReservedStateErrorMetadata.ErrorKind.PARSING, reservedState.errorMetadata().errorKind());
+                    assertThat(reservedState.errorMetadata().errors(), allOf(notNullValue(), hasSize(1)));
+                    assertThat(
+                        reservedState.errorMetadata().errors().get(0),
+                        containsString("failed to parse role-mapping [everyone_kibana_bad]. missing field [rules]")
+                    );
                 }
             }
         });
@@ -373,13 +325,22 @@ public class RoleMappingFileSettingsIT extends NativeRealmIntegTestCase {
         return new Tuple<>(savedClusterState, metadataVersion);
     }
 
+    private void assertRoleMappingsNotSaved(CountDownLatch savedClusterState, AtomicLong metadataVersion) throws Exception {
+        boolean awaitSuccessful = savedClusterState.await(20, TimeUnit.SECONDS);
+        assertTrue(awaitSuccessful);
+
+        // This should succeed, nothing was reserved
+        client().execute(PutRoleMappingAction.INSTANCE, sampleRestRequest("everyone_kibana_bad")).get();
+        client().execute(PutRoleMappingAction.INSTANCE, sampleRestRequest("everyone_fleet_ok")).get();
+    }
+
     public void testErrorSaved() throws Exception {
         ensureGreen();
 
         // save an empty file to clear any prior state, this ensures we don't get a stale file left over by another test
         var savedClusterState = setupClusterStateListenerForCleanup(internalCluster().getMasterName());
 
-        writeJSONFile(internalCluster().getMasterName(), emptyJSON, logger, versionCounter);
+        writeJSONFile(internalCluster().getMasterName(), emptyJSON);
         boolean awaitSuccessful = savedClusterState.v1().await(20, TimeUnit.SECONDS);
         assertTrue(awaitSuccessful);
 
@@ -392,94 +353,76 @@ public class RoleMappingFileSettingsIT extends NativeRealmIntegTestCase {
         );
 
         // save a bad file
-        savedClusterState = setupClusterStateListenerForError(
-            internalCluster().getCurrentMasterNodeInstance(ClusterService.class),
-            errorMetadata -> {
-                assertEquals(ReservedStateErrorMetadata.ErrorKind.PARSING, errorMetadata.errorKind());
-                assertThat(errorMetadata.errors(), allOf(notNullValue(), hasSize(1)));
-                assertThat(
-                    errorMetadata.errors().get(0),
-                    containsString("failed to parse role-mapping [everyone_kibana_bad]. missing field [rules]")
-                );
-            }
-        );
+        savedClusterState = setupClusterStateListenerForError(internalCluster().getMasterName());
 
-        writeJSONFile(internalCluster().getMasterName(), testErrorJSON, logger, versionCounter);
-        awaitSuccessful = savedClusterState.v1().await(20, TimeUnit.SECONDS);
-        assertTrue(awaitSuccessful);
+        writeJSONFile(internalCluster().getMasterName(), testErrorJSON);
+        assertRoleMappingsNotSaved(savedClusterState.v1(), savedClusterState.v2());
+    }
 
-        // no roles are resolved because both role mapping stores are empty
-        for (UserRoleMapper userRoleMapper : internalCluster().getInstances(UserRoleMapper.class)) {
-            PlainActionFuture<Set<String>> resolveRolesFuture = new PlainActionFuture<>();
-            userRoleMapper.resolveRoles(
-                new UserRoleMapper.UserData("anyUsername", null, List.of(), Map.of(), mock(RealmConfig.class)),
-                resolveRolesFuture
-            );
-            assertThat(resolveRolesFuture.get(), empty());
-        }
+    private Tuple<CountDownLatch, AtomicLong> setupClusterStateListenerForSecurityWriteError(String node) {
+        ClusterService clusterService = internalCluster().clusterService(node);
+        CountDownLatch savedClusterState = new CountDownLatch(1);
+        AtomicLong metadataVersion = new AtomicLong(-1);
+        clusterService.addListener(new ClusterStateListener() {
+            @Override
+            public void clusterChanged(ClusterChangedEvent event) {
+                ReservedStateMetadata reservedState = event.state().metadata().reservedStateMetadata().get(FileSettingsService.NAMESPACE);
+                if (reservedState != null
+                    && reservedState.errorMetadata() != null
+                    && reservedState.errorMetadata().errorKind() == ReservedStateErrorMetadata.ErrorKind.VALIDATION) {
+                    clusterService.removeListener(this);
+                    metadataVersion.set(event.state().metadata().version());
+                    savedClusterState.countDown();
+                    assertEquals(ReservedStateErrorMetadata.ErrorKind.VALIDATION, reservedState.errorMetadata().errorKind());
+                    assertThat(reservedState.errorMetadata().errors(), allOf(notNullValue(), hasSize(1)));
+                    assertThat(reservedState.errorMetadata().errors().get(0), containsString("closed"));
+                }
+            }
+        });
+
+        return new Tuple<>(savedClusterState, metadataVersion);
     }
 
-    public void testRoleMappingApplyWithSecurityIndexClosed() throws Exception {
+    public void testRoleMappingFailsToWriteToStore() throws Exception {
         ensureGreen();
 
-        // expect the role mappings to apply even if the .security index is closed
-        var savedClusterState = setupClusterStateListener(internalCluster().getMasterName(), "everyone_kibana");
+        var savedClusterState = setupClusterStateListenerForSecurityWriteError(internalCluster().getMasterName());
 
-        try {
-            var closeIndexResponse = indicesAdmin().close(new CloseIndexRequest(INTERNAL_SECURITY_MAIN_INDEX_7)).get();
-            assertTrue(closeIndexResponse.isAcknowledged());
-
-            writeJSONFile(internalCluster().getMasterName(), testJSON, logger, versionCounter);
-            boolean awaitSuccessful = savedClusterState.v1().await(20, TimeUnit.SECONDS);
-            assertTrue(awaitSuccessful);
-
-            // no native role mappings exist
-            var request = new GetRoleMappingsRequest();
-            request.setNames("everyone_kibana", "everyone_fleet");
-            var response = client().execute(GetRoleMappingsAction.INSTANCE, request).get();
-            assertFalse(response.hasMappings());
-
-            // cluster state settings are also applied
-            var clusterStateResponse = clusterAdmin().state(new ClusterStateRequest().waitForMetadataVersion(savedClusterState.v2().get()))
-                .get();
-            assertThat(
-                clusterStateResponse.getState().metadata().persistentSettings().get(INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey()),
-                equalTo("50mb")
-            );
-
-            ReservedStateMetadata reservedState = clusterStateResponse.getState()
-                .metadata()
-                .reservedStateMetadata()
-                .get(FileSettingsService.NAMESPACE);
-
-            ReservedStateHandlerMetadata handlerMetadata = reservedState.handlers().get(ReservedRoleMappingAction.NAME);
-            assertThat(handlerMetadata.keys(), containsInAnyOrder("everyone_kibana", "everyone_fleet"));
-
-            // and roles are resolved based on the cluster-state role mappings
-            for (UserRoleMapper userRoleMapper : internalCluster().getInstances(UserRoleMapper.class)) {
-                PlainActionFuture<Set<String>> resolveRolesFuture = new PlainActionFuture<>();
-                userRoleMapper.resolveRoles(
-                    new UserRoleMapper.UserData("anyUsername", null, List.of(), Map.of(), mock(RealmConfig.class)),
-                    resolveRolesFuture
-                );
-                assertThat(resolveRolesFuture.get(), containsInAnyOrder("kibana_user", "fleet_user"));
-            }
-        } finally {
-            savedClusterState = setupClusterStateListenerForCleanup(internalCluster().getMasterName());
-            writeJSONFile(internalCluster().getMasterName(), emptyJSON, logger, versionCounter);
-            boolean awaitSuccessful = savedClusterState.v1().await(20, TimeUnit.SECONDS);
-            assertTrue(awaitSuccessful);
-
-            var openIndexResponse = indicesAdmin().open(new OpenIndexRequest(INTERNAL_SECURITY_MAIN_INDEX_7)).get();
-            assertTrue(openIndexResponse.isAcknowledged());
-        }
+        final CloseIndexResponse closeIndexResponse = indicesAdmin().close(new CloseIndexRequest(INTERNAL_SECURITY_MAIN_INDEX_7)).get();
+        assertTrue(closeIndexResponse.isAcknowledged());
+
+        writeJSONFile(internalCluster().getMasterName(), testJSON);
+        boolean awaitSuccessful = savedClusterState.v1().await(20, TimeUnit.SECONDS);
+        assertTrue(awaitSuccessful);
+
+        var request = new GetRoleMappingsRequest();
+        request.setNames("everyone_kibana", "everyone_fleet");
+
+        var response = client().execute(GetRoleMappingsAction.INSTANCE, request).get();
+        assertFalse(response.hasMappings());
+
+        final ClusterStateResponse clusterStateResponse = clusterAdmin().state(
+            new ClusterStateRequest().waitForMetadataVersion(savedClusterState.v2().get())
+        ).get();
+
+        assertNull(
+            clusterStateResponse.getState().metadata().persistentSettings().get(INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey())
+        );
+
+        ReservedStateMetadata reservedState = clusterStateResponse.getState()
+            .metadata()
+            .reservedStateMetadata()
+            .get(FileSettingsService.NAMESPACE);
+
+        ReservedStateHandlerMetadata handlerMetadata = reservedState.handlers().get(ReservedRoleMappingAction.NAME);
+        assertTrue(handlerMetadata == null || handlerMetadata.keys().isEmpty());
     }
 
     private PutRoleMappingRequest sampleRestRequest(String name) throws Exception {
         var json = """
             {
-                "enabled": true,
-                "roles": [ "kibana_user_native" ],
+                "enabled": false,
+                "roles": [ "kibana_user" ],
                 "rules": { "field": { "username": "*" } },
                 "metadata": {
                     "uuid" : "b9a59ba9-6b92-4be2-bb8d-02bb270cb3a7"
@@ -490,7 +433,8 @@ public class RoleMappingFileSettingsIT extends NativeRealmIntegTestCase {
             var bis = new ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8));
             var parser = JSON.xContent().createParser(XContentParserConfiguration.EMPTY, bis)
         ) {
-            return new PutRoleMappingRequestBuilder(null).source(name, parser).request();
+            ExpressionRoleMapping mapping = ExpressionRoleMapping.parse(name, parser);
+            return PutRoleMappingRequest.fromMapping(mapping);
         }
     }
 }

x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/FileSettingsRoleMappingsStartupIT.java

@@ -0,0 +1,148 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security;
+
+import org.elasticsearch.analysis.common.CommonAnalysisPlugin;
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterStateListener;
+import org.elasticsearch.cluster.metadata.ReservedStateErrorMetadata;
+import org.elasticsearch.cluster.metadata.ReservedStateMetadata;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.core.Strings;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.reindex.ReindexPlugin;
+import org.elasticsearch.reservedstate.service.FileSettingsService;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.InternalSettingsPlugin;
+import org.elasticsearch.test.SecurityIntegTestCase;
+import org.elasticsearch.test.junit.annotations.TestLogging;
+import org.elasticsearch.transport.netty4.Netty4Plugin;
+import org.elasticsearch.xpack.wildcard.Wildcard;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static org.hamcrest.Matchers.allOf;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.notNullValue;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
+public class FileSettingsRoleMappingsStartupIT extends SecurityIntegTestCase {
+
+    private static AtomicLong versionCounter = new AtomicLong(1);
+    private static String testJSONForFailedCase = """
+        {
+             "metadata": {
+                 "version": "%s",
+                 "compatibility": "8.4.0"
+             },
+             "state": {
+                 "role_mappings": {
+                       "everyone_kibana_2": {
+                          "enabled": true,
+                          "roles": [ "kibana_user" ],
+                          "rules": { "field": { "username": "*" } },
+                          "metadata": {
+                             "uuid" : "b9a59ba9-6b92-4be2-bb8d-02bb270cb3a7",
+                             "_foo": "something"
+                          }
+                       }
+                 }
+             }
+        }""";
+
+    @Override
+    protected void doAssertXPackIsInstalled() {}
+
+    @Override
+    protected Path nodeConfigPath(int nodeOrdinal) {
+        return null;
+    }
+
+    private void writeJSONFile(String node, String json) throws Exception {
+        long version = versionCounter.incrementAndGet();
+
+        FileSettingsService fileSettingsService = internalCluster().getInstance(FileSettingsService.class, node);
+
+        Files.deleteIfExists(fileSettingsService.watchedFile());
+
+        Files.createDirectories(fileSettingsService.watchedFileDir());
+        Path tempFilePath = createTempFile();
+
+        logger.info("--> writing JSON config to node {} with path {}", node, tempFilePath);
+        logger.info(Strings.format(json, version));
+        Files.write(tempFilePath, Strings.format(json, version).getBytes(StandardCharsets.UTF_8));
+        Files.move(tempFilePath, fileSettingsService.watchedFile(), StandardCopyOption.ATOMIC_MOVE);
+    }
+
+    private Tuple<CountDownLatch, AtomicLong> setupClusterStateListenerForError(String node) {
+        ClusterService clusterService = internalCluster().clusterService(node);
+        CountDownLatch savedClusterState = new CountDownLatch(1);
+        AtomicLong metadataVersion = new AtomicLong(-1);
+        clusterService.addListener(new ClusterStateListener() {
+            @Override
+            public void clusterChanged(ClusterChangedEvent event) {
+                ReservedStateMetadata reservedState = event.state().metadata().reservedStateMetadata().get(FileSettingsService.NAMESPACE);
+                if (reservedState != null && reservedState.errorMetadata() != null) {
+                    assertEquals(ReservedStateErrorMetadata.ErrorKind.VALIDATION, reservedState.errorMetadata().errorKind());
+                    assertThat(reservedState.errorMetadata().errors(), allOf(notNullValue(), hasSize(1)));
+                    assertThat(reservedState.errorMetadata().errors().get(0), containsString("Fake exception"));
+                    clusterService.removeListener(this);
+                    metadataVersion.set(event.state().metadata().version());
+                    savedClusterState.countDown();
+                } else if (reservedState != null) {
+                    logger.debug(() -> "Got reserved state update without error metadata: " + reservedState);
+                } else {
+                    logger.debug(() -> "Got cluster state update: " + event.source());
+                }
+            }
+        });
+
+        return new Tuple<>(savedClusterState, metadataVersion);
+    }
+
+    @TestLogging(
+        value = "org.elasticsearch.common.file:DEBUG,org.elasticsearch.xpack.security:DEBUG,org.elasticsearch.cluster.metadata:DEBUG",
+        reason = "https://github.com/elastic/elasticsearch/issues/98391"
+    )
+    public void testFailsOnStartMasterNodeWithError() throws Exception {
+        internalCluster().setBootstrapMasterNodeIndex(0);
+
+        internalCluster().startMasterOnlyNode();
+
+        logger.info("--> write some role mappings, no other file settings");
+        writeJSONFile(internalCluster().getMasterName(), testJSONForFailedCase);
+        var savedClusterState = setupClusterStateListenerForError(internalCluster().getMasterName());
+
+        boolean awaitSuccessful = savedClusterState.v1().await(20, TimeUnit.SECONDS);
+        assertTrue(awaitSuccessful);
+    }
+
+    public Collection<Class<? extends Plugin>> nodePlugins() {
+        return Arrays.asList(
+            UnstableLocalStateSecurity.class,
+            Netty4Plugin.class,
+            ReindexPlugin.class,
+            CommonAnalysisPlugin.class,
+            InternalSettingsPlugin.class,
+            MapperExtrasPlugin.class,
+            Wildcard.class
+        );
+    }
+
+}

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java

@@ -1103,7 +1103,8 @@ public class Security extends Plugin
             new SecurityUsageServices(realms, allRolesStore, nativeRoleMappingStore, ipFilter.get(), profileService, apiKeyService)
         );
 
-        reservedRoleMappingAction.set(new ReservedRoleMappingAction());
+        reservedRoleMappingAction.set(new ReservedRoleMappingAction(nativeRoleMappingStore));
+        systemIndices.getMainIndexManager().onStateRecovered(state -> reservedRoleMappingAction.get().securityIndexRecovered());
 
         cacheInvalidatorRegistry.validate();
 

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/ReservedRoleMappingAction.java

@@ -7,18 +7,24 @@
 
 package org.elasticsearch.xpack.security.action.rolemapping;
 
-import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.GroupedActionListener;
+import org.elasticsearch.common.util.concurrent.ListenableFuture;
+import org.elasticsearch.reservedstate.NonStateTransformResult;
 import org.elasticsearch.reservedstate.ReservedClusterStateHandler;
 import org.elasticsearch.reservedstate.TransformState;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentParserConfiguration;
+import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingRequest;
 import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingRequest;
-import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingRequestBuilder;
 import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping;
-import org.elasticsearch.xpack.core.security.authz.RoleMappingMetadata;
+import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -32,59 +38,123 @@ import static org.elasticsearch.common.xcontent.XContentHelper.mapToXContentPars
  * It is used by the ReservedClusterStateService to add/update or remove role mappings. Typical usage
  * for this action is in the context of file based settings.
  */
-public class ReservedRoleMappingAction implements ReservedClusterStateHandler<List<PutRoleMappingRequest>> {
+public class ReservedRoleMappingAction implements ReservedClusterStateHandler<List<ExpressionRoleMapping>> {
     public static final String NAME = "role_mappings";
 
+    private final NativeRoleMappingStore roleMappingStore;
+    private final ListenableFuture<Void> securityIndexRecoveryListener = new ListenableFuture<>();
+
+    /**
+     * Creates a ReservedRoleMappingAction
+     *
+     * @param roleMappingStore requires {@link NativeRoleMappingStore} for storing/deleting the mappings
+     */
+    public ReservedRoleMappingAction(NativeRoleMappingStore roleMappingStore) {
+        this.roleMappingStore = roleMappingStore;
+    }
+
     @Override
     public String name() {
         return NAME;
     }
 
+    private static Collection<PutRoleMappingRequest> prepare(List<ExpressionRoleMapping> roleMappings) {
+        List<PutRoleMappingRequest> requests = roleMappings.stream().map(rm -> PutRoleMappingRequest.fromMapping(rm)).toList();
+
+        var exceptions = new ArrayList<Exception>();
+        for (var request : requests) {
+            // File based defined role mappings are allowed to use MetadataUtils.RESERVED_PREFIX
+            var exception = request.validate(false);
+            if (exception != null) {
+                exceptions.add(exception);
+            }
+        }
+
+        if (exceptions.isEmpty() == false) {
+            var illegalArgumentException = new IllegalArgumentException("error on validating put role mapping requests");
+            exceptions.forEach(illegalArgumentException::addSuppressed);
+            throw illegalArgumentException;
+        }
+
+        return requests;
+    }
+
     @Override
     public TransformState transform(Object source, TransformState prevState) throws Exception {
+        // We execute the prepare() call to catch any errors in the transform phase.
+        // Since we store the role mappings outside the cluster state, we do the actual save with a
+        // non cluster state transform call.
         @SuppressWarnings("unchecked")
-        Set<ExpressionRoleMapping> roleMappings = validate((List<PutRoleMappingRequest>) source);
-        RoleMappingMetadata newRoleMappingMetadata = new RoleMappingMetadata(roleMappings);
-        if (newRoleMappingMetadata.equals(RoleMappingMetadata.getFromClusterState(prevState.state()))) {
-            return prevState;
-        } else {
-            ClusterState newState = newRoleMappingMetadata.updateClusterState(prevState.state());
-            Set<String> entities = newRoleMappingMetadata.getRoleMappings()
-                .stream()
-                .map(ExpressionRoleMapping::getName)
-                .collect(Collectors.toSet());
-            return new TransformState(newState, entities);
+        var requests = prepare((List<ExpressionRoleMapping>) source);
+        return new TransformState(
+            prevState.state(),
+            prevState.keys(),
+            l -> securityIndexRecoveryListener.addListener(
+                ActionListener.wrap(ignored -> nonStateTransform(requests, prevState, l), l::onFailure)
+            )
+        );
+    }
+
+    // Exposed for testing purposes
+    protected void nonStateTransform(
+        Collection<PutRoleMappingRequest> requests,
+        TransformState prevState,
+        ActionListener<NonStateTransformResult> listener
+    ) {
+        Set<String> entities = requests.stream().map(r -> r.getName()).collect(Collectors.toSet());
+        Set<String> toDelete = new HashSet<>(prevState.keys());
+        toDelete.removeAll(entities);
+
+        final int tasksCount = requests.size() + toDelete.size();
+
+        // Nothing to do, don't start a group listener with 0 actions
+        if (tasksCount == 0) {
+            listener.onResponse(new NonStateTransformResult(ReservedRoleMappingAction.NAME, Set.of()));
+            return;
+        }
+
+        GroupedActionListener<Boolean> taskListener = new GroupedActionListener<>(tasksCount, new ActionListener<>() {
+            @Override
+            public void onResponse(Collection<Boolean> booleans) {
+                listener.onResponse(new NonStateTransformResult(ReservedRoleMappingAction.NAME, Collections.unmodifiableSet(entities)));
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                listener.onFailure(e);
+            }
+        });
+
+        for (var request : requests) {
+            roleMappingStore.putRoleMapping(request, taskListener);
+        }
+
+        for (var mappingToDelete : toDelete) {
+            var deleteRequest = new DeleteRoleMappingRequest();
+            deleteRequest.setName(mappingToDelete);
+            roleMappingStore.deleteRoleMapping(deleteRequest, taskListener);
         }
     }
 
     @Override
-    public List<PutRoleMappingRequest> fromXContent(XContentParser parser) throws IOException {
-        List<PutRoleMappingRequest> result = new ArrayList<>();
+    public List<ExpressionRoleMapping> fromXContent(XContentParser parser) throws IOException {
+        List<ExpressionRoleMapping> result = new ArrayList<>();
+
         Map<String, ?> source = parser.map();
+
         for (String name : source.keySet()) {
             @SuppressWarnings("unchecked")
             Map<String, ?> content = (Map<String, ?>) source.get(name);
             try (XContentParser mappingParser = mapToXContentParser(XContentParserConfiguration.EMPTY, content)) {
-                result.add(new PutRoleMappingRequestBuilder(null).source(name, mappingParser).request());
+                ExpressionRoleMapping mapping = ExpressionRoleMapping.parse(name, mappingParser);
+                result.add(mapping);
             }
         }
+
         return result;
     }
 
-    private Set<ExpressionRoleMapping> validate(List<PutRoleMappingRequest> roleMappings) {
-        var exceptions = new ArrayList<Exception>();
-        for (var roleMapping : roleMappings) {
-            // File based defined role mappings are allowed to use MetadataUtils.RESERVED_PREFIX
-            var exception = roleMapping.validate(false);
-            if (exception != null) {
-                exceptions.add(exception);
-            }
-        }
-        if (exceptions.isEmpty() == false) {
-            var illegalArgumentException = new IllegalArgumentException("error on validating put role mapping requests");
-            exceptions.forEach(illegalArgumentException::addSuppressed);
-            throw illegalArgumentException;
-        }
-        return roleMappings.stream().map(PutRoleMappingRequest::getMapping).collect(Collectors.toUnmodifiableSet());
+    public void securityIndexRecovered() {
+        securityIndexRecoveryListener.onResponse(null);
     }
 }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportDeleteRoleMappingAction.java

@@ -8,9 +8,9 @@ package org.elasticsearch.xpack.security.action.rolemapping;
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.action.support.ReservedStateAwareHandledTransportAction;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingAction;
@@ -18,7 +18,12 @@ import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappin
 import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingResponse;
 import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore;
 
-public class TransportDeleteRoleMappingAction extends HandledTransportAction<DeleteRoleMappingRequest, DeleteRoleMappingResponse> {
+import java.util.Optional;
+import java.util.Set;
+
+public class TransportDeleteRoleMappingAction extends ReservedStateAwareHandledTransportAction<
+    DeleteRoleMappingRequest,
+    DeleteRoleMappingResponse> {
 
     private final NativeRoleMappingStore roleMappingStore;
 
@@ -26,20 +31,25 @@ public class TransportDeleteRoleMappingAction extends HandledTransportAction<Del
     public TransportDeleteRoleMappingAction(
         ActionFilters actionFilters,
         TransportService transportService,
+        ClusterService clusterService,
         NativeRoleMappingStore roleMappingStore
     ) {
-        super(
-            DeleteRoleMappingAction.NAME,
-            transportService,
-            actionFilters,
-            DeleteRoleMappingRequest::new,
-            EsExecutors.DIRECT_EXECUTOR_SERVICE
-        );
+        super(DeleteRoleMappingAction.NAME, clusterService, transportService, actionFilters, DeleteRoleMappingRequest::new);
         this.roleMappingStore = roleMappingStore;
     }
 
     @Override
-    protected void doExecute(Task task, DeleteRoleMappingRequest request, ActionListener<DeleteRoleMappingResponse> listener) {
+    protected void doExecuteProtected(Task task, DeleteRoleMappingRequest request, ActionListener<DeleteRoleMappingResponse> listener) {
         roleMappingStore.deleteRoleMapping(request, listener.safeMap(DeleteRoleMappingResponse::new));
     }
+
+    @Override
+    public Optional<String> reservedStateHandlerName() {
+        return Optional.of(ReservedRoleMappingAction.NAME);
+    }
+
+    @Override
+    public Set<String> modifiedKeys(DeleteRoleMappingRequest request) {
+        return Set.of(request.getName());
+    }
 }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportPutRoleMappingAction.java

@@ -8,9 +8,9 @@ package org.elasticsearch.xpack.security.action.rolemapping;
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.action.support.ReservedStateAwareHandledTransportAction;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingAction;
@@ -18,7 +18,10 @@ import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingRe
 import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingResponse;
 import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore;
 
-public class TransportPutRoleMappingAction extends HandledTransportAction<PutRoleMappingRequest, PutRoleMappingResponse> {
+import java.util.Optional;
+import java.util.Set;
+
+public class TransportPutRoleMappingAction extends ReservedStateAwareHandledTransportAction<PutRoleMappingRequest, PutRoleMappingResponse> {
 
     private final NativeRoleMappingStore roleMappingStore;
 
@@ -26,17 +29,32 @@ public class TransportPutRoleMappingAction extends HandledTransportAction<PutRol
     public TransportPutRoleMappingAction(
         ActionFilters actionFilters,
         TransportService transportService,
+        ClusterService clusterService,
         NativeRoleMappingStore roleMappingStore
     ) {
-        super(PutRoleMappingAction.NAME, transportService, actionFilters, PutRoleMappingRequest::new, EsExecutors.DIRECT_EXECUTOR_SERVICE);
+        super(PutRoleMappingAction.NAME, clusterService, transportService, actionFilters, PutRoleMappingRequest::new);
         this.roleMappingStore = roleMappingStore;
     }
 
     @Override
-    protected void doExecute(Task task, final PutRoleMappingRequest request, final ActionListener<PutRoleMappingResponse> listener) {
+    protected void doExecuteProtected(
+        Task task,
+        final PutRoleMappingRequest request,
+        final ActionListener<PutRoleMappingResponse> listener
+    ) {
         roleMappingStore.putRoleMapping(
             request,
             ActionListener.wrap(created -> listener.onResponse(new PutRoleMappingResponse(created)), listener::onFailure)
         );
     }
+
+    @Override
+    public Optional<String> reservedStateHandlerName() {
+        return Optional.of(ReservedRoleMappingAction.NAME);
+    }
+
+    @Override
+    public Set<String> modifiedKeys(PutRoleMappingRequest request) {
+        return Set.of(request.getName());
+    }
 }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/rolemapping/RestPutRoleMappingAction.java

@@ -8,8 +8,6 @@ package org.elasticsearch.xpack.security.rest.action.rolemapping;
 
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
-import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestRequest;
@@ -19,7 +17,6 @@ import org.elasticsearch.rest.Scope;
 import org.elasticsearch.rest.ServerlessScope;
 import org.elasticsearch.rest.action.RestBuilderListener;
 import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingRequestBuilder;
 import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingResponse;
 
@@ -60,18 +57,12 @@ public class RestPutRoleMappingAction extends NativeRoleMappingBaseRestHandler {
 
     @Override
     public RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException {
-        String name = request.param("name");
-        String refresh = request.param("refresh");
-        PutRoleMappingRequestBuilder requestBuilder;
-        try (
-            XContentParser parser = XContentHelper.createParserNotCompressed(
-                LoggingDeprecationHandler.XCONTENT_PARSER_CONFIG,
-                request.requiredContent(),
-                request.getXContentType()
-            )
-        ) {
-            requestBuilder = new PutRoleMappingRequestBuilder(client).source(name, parser).setRefreshPolicy(refresh);
-        }
+        final String name = request.param("name");
+        PutRoleMappingRequestBuilder requestBuilder = new PutRoleMappingRequestBuilder(client).source(
+            name,
+            request.requiredContent(),
+            request.getXContentType()
+        ).setRefreshPolicy(request.param("refresh"));
         return channel -> requestBuilder.execute(new RestBuilderListener<>(channel) {
             @Override
             public RestResponse buildResponse(PutRoleMappingResponse response, XContentBuilder builder) throws Exception {

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/LocalReservedUnstableSecurityStateHandlerProvider.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security;
+
+import org.elasticsearch.reservedstate.ReservedClusterStateHandlerProvider;
+
+/**
+ * Mock Security Provider implementation for the {@link ReservedClusterStateHandlerProvider} service interface. This is used
+ * for {@link org.elasticsearch.test.ESIntegTestCase} because the Security Plugin is really LocalStateSecurity in those tests.
+ * <p>
+ * Unlike {@link LocalReservedSecurityStateHandlerProvider} this implementation is mocked to implement the
+ * {@link UnstableLocalStateSecurity}. Separate implementation is needed, because the SPI creation code matches the constructor
+ * signature when instantiating. E.g. we need to match {@link UnstableLocalStateSecurity} instead of {@link LocalStateSecurity}
+ */
+public class LocalReservedUnstableSecurityStateHandlerProvider extends LocalReservedSecurityStateHandlerProvider {
+    public LocalReservedUnstableSecurityStateHandlerProvider() {
+        throw new IllegalStateException("Provider must be constructed using PluginsService");
+    }
+
+    public LocalReservedUnstableSecurityStateHandlerProvider(UnstableLocalStateSecurity plugin) {
+        super(plugin);
+    }
+}

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/UnstableLocalStateSecurity.java

@@ -0,0 +1,97 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.reservedstate.NonStateTransformResult;
+import org.elasticsearch.reservedstate.ReservedClusterStateHandler;
+import org.elasticsearch.reservedstate.TransformState;
+import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingRequest;
+import org.elasticsearch.xpack.core.ssl.SSLService;
+import org.elasticsearch.xpack.security.action.rolemapping.ReservedRoleMappingAction;
+
+import java.nio.file.Path;
+import java.util.Collection;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * A test class that allows us to Inject new type of Reserved Handler that can
+ * simulate errors in saving role mappings.
+ * <p>
+ * We can't use our regular path to simply make an extension of LocalStateSecurity
+ * in an integration test class, because the reserved handlers are injected through
+ * SPI. (see {@link LocalReservedUnstableSecurityStateHandlerProvider})
+ */
+public final class UnstableLocalStateSecurity extends LocalStateSecurity {
+
+    public UnstableLocalStateSecurity(Settings settings, Path configPath) throws Exception {
+        super(settings, configPath);
+        // We reuse most of the initialization of LocalStateSecurity, we then just overwrite
+        // the security plugin with an extra method to give us a fake RoleMappingAction.
+        Optional<Plugin> security = plugins.stream().filter(p -> p instanceof Security).findFirst();
+        if (security.isPresent()) {
+            plugins.remove(security.get());
+        }
+
+        UnstableLocalStateSecurity thisVar = this;
+        var action = new ReservedUnstableRoleMappingAction();
+
+        plugins.add(new Security(settings, super.securityExtensions()) {
+            @Override
+            protected SSLService getSslService() {
+                return thisVar.getSslService();
+            }
+
+            @Override
+            protected XPackLicenseState getLicenseState() {
+                return thisVar.getLicenseState();
+            }
+
+            @Override
+            List<ReservedClusterStateHandler<?>> reservedClusterStateHandlers() {
+                // pretend the security index is initialized after 2 seconds
+                var timer = new java.util.Timer();
+                timer.schedule(new java.util.TimerTask() {
+                    @Override
+                    public void run() {
+                        action.securityIndexRecovered();
+                        timer.cancel();
+                    }
+                }, 2_000);
+                return List.of(action);
+            }
+        });
+    }
+
+    public static class ReservedUnstableRoleMappingAction extends ReservedRoleMappingAction {
+        /**
+         * Creates a fake ReservedRoleMappingAction that doesn't actually use the role mapping store
+         */
+        public ReservedUnstableRoleMappingAction() {
+            // we don't actually need a NativeRoleMappingStore
+            super(null);
+        }
+
+        /**
+         * The nonStateTransform method is the only one that uses the native store, we simply pretend
+         * something has called the onFailure method of the listener.
+         */
+        @Override
+        protected void nonStateTransform(
+            Collection<PutRoleMappingRequest> requests,
+            TransformState prevState,
+            ActionListener<NonStateTransformResult> listener
+        ) {
+            listener.onFailure(new IllegalStateException("Fake exception"));
+        }
+    }
+}

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/reservedstate/ReservedRoleMappingActionTests.java

@@ -7,40 +7,77 @@
 
 package org.elasticsearch.xpack.security.action.reservedstate;
 
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.reservedstate.NonStateTransformResult;
 import org.elasticsearch.reservedstate.TransformState;
+import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentParserConfiguration;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.security.action.rolemapping.ReservedRoleMappingAction;
+import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore;
+import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 
 import java.util.Collections;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicReference;
 
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.empty;
-import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
 
 /**
  * Tests that the ReservedRoleMappingAction does validation, can add and remove role mappings
  */
 public class ReservedRoleMappingActionTests extends ESTestCase {
-
     private TransformState processJSON(ReservedRoleMappingAction action, TransformState prevState, String json) throws Exception {
         try (XContentParser parser = XContentType.JSON.xContent().createParser(XContentParserConfiguration.EMPTY, json)) {
             var content = action.fromXContent(parser);
             var state = action.transform(content, prevState);
-            assertThat(state.nonStateTransform(), nullValue());
-            return state;
+
+            CountDownLatch latch = new CountDownLatch(1);
+            AtomicReference<Set<String>> updatedKeys = new AtomicReference<>();
+            AtomicReference<Exception> error = new AtomicReference<>();
+            state.nonStateTransform().accept(new ActionListener<>() {
+                @Override
+                public void onResponse(NonStateTransformResult nonStateTransformResult) {
+                    updatedKeys.set(nonStateTransformResult.updatedKeys());
+                    latch.countDown();
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    error.set(e);
+                    latch.countDown();
+                }
+            });
+
+            latch.await();
+            if (error.get() != null) {
+                throw error.get();
+            }
+            return new TransformState(state.state(), updatedKeys.get());
         }
     }
 
     public void testValidation() {
+        var nativeRoleMappingStore = mockNativeRoleMappingStore();
+
         ClusterState state = ClusterState.builder(new ClusterName("elasticsearch")).build();
         TransformState prevState = new TransformState(state, Collections.emptySet());
-        ReservedRoleMappingAction action = new ReservedRoleMappingAction();
+        ReservedRoleMappingAction action = new ReservedRoleMappingAction(nativeRoleMappingStore);
+        action.securityIndexRecovered();
+
         String badPolicyJSON = """
             {
                "everyone_kibana": {
@@ -60,6 +97,7 @@ public class ReservedRoleMappingActionTests extends ESTestCase {
                   }
                }
             }""";
+
         assertEquals(
             "failed to parse role-mapping [everyone_fleet]. missing field [rules]",
             expectThrows(ParsingException.class, () -> processJSON(action, prevState, badPolicyJSON)).getMessage()
@@ -67,9 +105,13 @@ public class ReservedRoleMappingActionTests extends ESTestCase {
     }
 
     public void testAddRemoveRoleMapping() throws Exception {
+        var nativeRoleMappingStore = mockNativeRoleMappingStore();
+
         ClusterState state = ClusterState.builder(new ClusterName("elasticsearch")).build();
         TransformState prevState = new TransformState(state, Collections.emptySet());
-        ReservedRoleMappingAction action = new ReservedRoleMappingAction();
+        ReservedRoleMappingAction action = new ReservedRoleMappingAction(nativeRoleMappingStore);
+        action.securityIndexRecovered();
+
         String emptyJSON = "";
 
         TransformState updatedState = processJSON(action, prevState, emptyJSON);
@@ -105,4 +147,102 @@ public class ReservedRoleMappingActionTests extends ESTestCase {
         updatedState = processJSON(action, prevState, emptyJSON);
         assertThat(updatedState.keys(), empty());
     }
+
+    @SuppressWarnings("unchecked")
+    public void testNonStateTransformWaitsOnAsyncActions() throws Exception {
+        var nativeRoleMappingStore = mockNativeRoleMappingStore();
+
+        doAnswer(invocation -> {
+            new Thread(() -> {
+                // Simulate put role mapping async action taking a while
+                try {
+                    Thread.sleep(1_000);
+                    ((ActionListener<Boolean>) invocation.getArgument(1)).onFailure(new IllegalStateException("err_done"));
+                } catch (InterruptedException e) {
+                    throw new RuntimeException(e);
+                }
+            }).start();
+
+            return null;
+        }).when(nativeRoleMappingStore).putRoleMapping(any(), any());
+
+        doAnswer(invocation -> {
+            new Thread(() -> {
+                // Simulate delete role mapping async action taking a while
+                try {
+                    Thread.sleep(1_000);
+                    ((ActionListener<Boolean>) invocation.getArgument(1)).onFailure(new IllegalStateException("err_done"));
+                } catch (InterruptedException e) {
+                    throw new RuntimeException(e);
+                }
+            }).start();
+
+            return null;
+        }).when(nativeRoleMappingStore).deleteRoleMapping(any(), any());
+
+        ClusterState state = ClusterState.builder(new ClusterName("elasticsearch")).build();
+        TransformState updatedState = new TransformState(state, Collections.emptySet());
+        ReservedRoleMappingAction action = new ReservedRoleMappingAction(nativeRoleMappingStore);
+        action.securityIndexRecovered();
+
+        String json = """
+            {
+               "everyone_kibana": {
+                  "enabled": true,
+                  "roles": [ "kibana_user" ],
+                  "rules": { "field": { "username": "*" } },
+                  "metadata": {
+                     "uuid" : "b9a59ba9-6b92-4be2-bb8d-02bb270cb3a7",
+                     "_reserved": true
+                  }
+               },
+               "everyone_fleet": {
+                  "enabled": true,
+                  "roles": [ "fleet_user" ],
+                  "rules": { "field": { "username": "*" } },
+                  "metadata": {
+                     "uuid" : "a9a59ba9-6b92-4be2-bb8d-02bb270cb3a7",
+                     "_reserved": true
+                  }
+               }
+            }""";
+
+        assertEquals(
+            "err_done",
+            expectThrows(IllegalStateException.class, () -> processJSON(action, new TransformState(state, Collections.emptySet()), json))
+                .getMessage()
+        );
+
+        // Now that we've tested that we wait on putRoleMapping correctly, let it finish without exception, so we can test error on delete
+        doAnswer(invocation -> {
+            ((ActionListener<Boolean>) invocation.getArgument(1)).onResponse(true);
+            return null;
+        }).when(nativeRoleMappingStore).putRoleMapping(any(), any());
+
+        updatedState = processJSON(action, updatedState, json);
+        assertThat(updatedState.keys(), containsInAnyOrder("everyone_kibana", "everyone_fleet"));
+
+        final TransformState currentState = new TransformState(updatedState.state(), updatedState.keys());
+
+        assertEquals("err_done", expectThrows(IllegalStateException.class, () -> processJSON(action, currentState, "")).getMessage());
+    }
+
+    @SuppressWarnings("unchecked")
+    private NativeRoleMappingStore mockNativeRoleMappingStore() {
+        final NativeRoleMappingStore nativeRoleMappingStore = spy(
+            new NativeRoleMappingStore(Settings.EMPTY, mock(Client.class), mock(SecurityIndexManager.class), mock(ScriptService.class))
+        );
+
+        doAnswer(invocation -> {
+            ((ActionListener<Boolean>) invocation.getArgument(1)).onResponse(true);
+            return null;
+        }).when(nativeRoleMappingStore).putRoleMapping(any(), any());
+
+        doAnswer(invocation -> {
+            ((ActionListener<Boolean>) invocation.getArgument(1)).onResponse(true);
+            return null;
+        }).when(nativeRoleMappingStore).deleteRoleMapping(any(), any());
+
+        return nativeRoleMappingStore;
+    }
 }

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/rolemapping/TransportDeleteRoleMappingActionTests.java

@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.action.rolemapping;
+
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.Transport;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingRequest;
+import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore;
+
+import java.util.Collections;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.mockito.Mockito.mock;
+
+public class TransportDeleteRoleMappingActionTests extends ESTestCase {
+    public void testReservedStateHandler() {
+        var store = mock(NativeRoleMappingStore.class);
+        TransportService transportService = new TransportService(
+            Settings.EMPTY,
+            mock(Transport.class),
+            mock(ThreadPool.class),
+            TransportService.NOOP_TRANSPORT_INTERCEPTOR,
+            x -> null,
+            null,
+            Collections.emptySet()
+        );
+        var action = new TransportDeleteRoleMappingAction(mock(ActionFilters.class), transportService, mock(ClusterService.class), store);
+
+        assertEquals(ReservedRoleMappingAction.NAME, action.reservedStateHandlerName().get());
+
+        var deleteRequest = new DeleteRoleMappingRequest();
+        deleteRequest.setName("kibana_all");
+        assertThat(action.modifiedKeys(deleteRequest), containsInAnyOrder("kibana_all"));
+    }
+}

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/rolemapping/TransportPutRoleMappingActionTests.java

@@ -9,12 +9,16 @@ package org.elasticsearch.xpack.security.action.rolemapping;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
+import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingRequest;
 import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingResponse;
 import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping;
@@ -29,6 +33,7 @@ import java.util.concurrent.atomic.AtomicReference;
 
 import static org.hamcrest.Matchers.aMapWithSize;
 import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.iterableWithSize;
@@ -55,7 +60,7 @@ public class TransportPutRoleMappingActionTests extends ESTestCase {
             null,
             Collections.emptySet()
         );
-        action = new TransportPutRoleMappingAction(mock(ActionFilters.class), transportService, store);
+        action = new TransportPutRoleMappingAction(mock(ActionFilters.class), transportService, mock(ClusterService.class), store);
 
         requestRef = new AtomicReference<>(null);
 
@@ -94,7 +99,39 @@ public class TransportPutRoleMappingActionTests extends ESTestCase {
         request.setMetadata(metadata);
         request.setEnabled(true);
         final PlainActionFuture<PutRoleMappingResponse> future = new PlainActionFuture<>();
-        action.doExecute(mock(Task.class), request, future);
+        action.doExecuteProtected(mock(Task.class), request, future);
         return future.get();
     }
+
+    public void testReservedStateHandler() throws Exception {
+        assertEquals(ReservedRoleMappingAction.NAME, action.reservedStateHandlerName().get());
+        String json = """
+            {
+               "everyone_kibana": {
+                  "enabled": true,
+                  "roles": [ "kibana_user" ],
+                  "rules": { "field": { "username": "*" } },
+                  "metadata": {
+                     "uuid" : "b9a59ba9-6b92-4be2-bb8d-02bb270cb3a7"
+                  }
+               },
+               "everyone_fleet": {
+                  "enabled": true,
+                  "roles": [ "fleet_user" ],
+                  "rules": { "field": { "username": "*" } },
+                  "metadata": {
+                     "uuid" : "b9a59ba9-6b92-4be3-bb8d-02bb270cb3a7"
+                  }
+               }
+            }""";
+
+        try (XContentParser parser = XContentType.JSON.xContent().createParser(XContentParserConfiguration.EMPTY, json)) {
+            ReservedRoleMappingAction roleMappingAction = new ReservedRoleMappingAction(store);
+            var parsedResult = roleMappingAction.fromXContent(parser);
+
+            for (var mapping : parsedResult) {
+                assertThat(action.modifiedKeys(PutRoleMappingRequest.fromMapping(mapping)), containsInAnyOrder(mapping.getName()));
+            }
+        }
+    }
 }

x-pack/plugin/security/src/test/resources/META-INF/services/org.elasticsearch.reservedstate.ReservedClusterStateHandlerProvider

@@ -6,3 +6,4 @@
 #
 
 org.elasticsearch.xpack.security.LocalReservedSecurityStateHandlerProvider
+org.elasticsearch.xpack.security.LocalReservedUnstableSecurityStateHandlerProvider

x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java

@@ -20,14 +20,11 @@ import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.ssl.SslVerificationMode;
 import org.elasticsearch.common.util.Maps;
-import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
-import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.SecurityIntegTestCase;
 import org.elasticsearch.test.fixtures.smb.SmbTestContainer;
 import org.elasticsearch.test.fixtures.testcontainers.TestContainersThreadFilter;
-import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingRequestBuilder;
 import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingResponse;
@@ -190,16 +187,11 @@ public abstract class AbstractAdLdapRealmTestCase extends SecurityIntegTestCase
         Map<String, ActionFuture<PutRoleMappingResponse>> futures = Maps.newLinkedHashMapWithExpectedSize(content.size());
         for (int i = 0; i < content.size(); i++) {
             final String name = "external_" + i;
-            final PutRoleMappingRequestBuilder builder;
-            try (
-                XContentParser parser = XContentHelper.createParserNotCompressed(
-                    LoggingDeprecationHandler.XCONTENT_PARSER_CONFIG,
-                    new BytesArray(content.get(i)),
-                    XContentType.JSON
-                )
-            ) {
-                builder = new PutRoleMappingRequestBuilder(client()).source(name, parser);
-            }
+            final PutRoleMappingRequestBuilder builder = new PutRoleMappingRequestBuilder(client()).source(
+                name,
+                new BytesArray(content.get(i)),
+                XContentType.JSON
+            );
             futures.put(name, builder.execute());
         }
         for (String mappingName : futures.keySet()) {