[8.x] Enable queryable built-in roles feature by default (#120323) (#120886)

* Enable queryable built-in roles feature by default (#120323)

Making the `es.queryable_built_in_roles_enabled` feature flag enabled by default.
This feature makes the built-in roles automatically indexed in `.security` index and available
for querying via Query Role API. The consequence of this is that `.security` index is now
created eagerly (if it's not existing) on cluster formation.

In order to keep the scope of this PR small, the feature is disabled for some of the tests,
because they are either non-trivial to adjust or the gain is not worthy the effort to do it now.
The tests will be adjusted in a follow-up PR and later the flag will be removed completely.

Relates to #117581

(cherry picked from commit 52e0f21bdd0e6446c8de380b58d13d880b430dbd)

# Conflicts:
#	modules/dot-prefix-validation/build.gradle
#	test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
#	x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmElasticAutoconfigIntegTests.java

* Update InternalTestCluster.java

remove line snuck after resolving merge confilcs

* Update build.gradle

fix build.gradle

* Update build.gradle

fix build.gradle by removing invalid task

* remove non-existing timeout parameter on 8.x branch

docs/build.gradle

@@ -120,6 +120,8 @@ testClusters.matching { it.name == "yamlRestTest"}.configureEach {
   // TODO: remove this once cname is prepended to transport.publish_address by default in 8.0
   systemProperty 'es.transport.cname_in_publish_address', 'true'
 
+  systemProperty 'es.queryable_built_in_roles_enabled', 'false'
+
   requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0")
   requiresFeature 'es.failure_store_feature_flag_enabled', Version.fromString("8.12.0")
 

modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/140_data_stream_aliases.yml

@@ -240,7 +240,8 @@
             test: {}
 
   - do:
-      indices.get_alias: { }
+      indices.get_alias:
+        index: test*
   - match: { test1.aliases.test: { } }
   - match: { test2.aliases.test: { } }
   - match: { test3.aliases.test: { } }
@@ -255,7 +256,8 @@
   - is_true: acknowledged
 
   - do:
-      indices.get_alias: {}
+      indices.get_alias:
+        index: test*
   - match: {test1.aliases: {}}
   - match: {test2.aliases: {}}
   - match: {test3.aliases: {}}

modules/dot-prefix-validation/src/yamlRestTest/java/org/elasticsearch/validation/DotPrefixClientYamlTestSuiteIT.java

@@ -21,6 +21,8 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 import org.junit.ClassRule;
 
+import java.util.Objects;
+
 import static org.elasticsearch.test.cluster.FeatureFlag.FAILURE_STORE_ENABLED;
 
 public class DotPrefixClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
@@ -55,6 +57,10 @@ public class DotPrefixClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
         if (setNodes) {
             clusterBuilder.nodes(2);
         }
+        clusterBuilder.systemProperty("es.queryable_built_in_roles_enabled", () -> {
+            final String enabled = System.getProperty("es.queryable_built_in_roles_enabled");
+            return Objects.requireNonNullElse(enabled, "");
+        });
         return clusterBuilder.build();
     }
 

modules/dot-prefix-validation/src/yamlRestTest/resources/rest-api-spec/test/dot_prefix/10_basic.yml

@@ -2,7 +2,7 @@
 teardown:
   - do:
       indices.delete:
-        index: .*
+        index: .*,-.security-*
 
 ---
 "Index creation with a dot-prefix is deprecated unless x-elastic-product-origin set":

qa/packaging/src/test/java/org/elasticsearch/packaging/test/PasswordToolsTests.java

@@ -20,6 +20,7 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.Callable;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Stream;
@@ -47,7 +48,9 @@ public class PasswordToolsTests extends PackagingTestCase {
     public void test20GeneratePasswords() throws Exception {
         assertWhileRunning(() -> {
             ServerUtils.waitForElasticsearch(installation);
-            Shell.Result result = installation.executables().setupPasswordsTool.run("auto --batch", null);
+            Shell.Result result = retryOnAuthenticationErrors(
+                () -> installation.executables().setupPasswordsTool.run("auto --batch", null)
+            );
             Map<String, String> userpasses = parseUsersAndPasswords(result.stdout());
             for (Map.Entry<String, String> userpass : userpasses.entrySet()) {
                 String response = ServerUtils.makeRequest(
@@ -102,20 +105,26 @@ public class PasswordToolsTests extends PackagingTestCase {
         installation.executables().keystoreTool.run("add --stdin bootstrap.password", BOOTSTRAP_PASSWORD);
 
         assertWhileRunning(() -> {
-            String response = ServerUtils.makeRequest(
-                Request.Get("http://localhost:9200/_cluster/health?wait_for_status=green&timeout=180s"),
-                "elastic",
-                BOOTSTRAP_PASSWORD,
-                null
+            ServerUtils.waitForElasticsearch("green", null, installation, "elastic", BOOTSTRAP_PASSWORD, null);
+            final String response = retryOnAuthenticationErrors(
+                () -> ServerUtils.makeRequest(
+                    Request.Get("http://localhost:9200/_cluster/health?wait_for_status=green&timeout=180s"),
+                    "elastic",
+                    BOOTSTRAP_PASSWORD,
+                    null
+                )
             );
             assertThat(response, containsString("\"status\":\"green\""));
         });
+
     }
 
     public void test40GeneratePasswordsBootstrapAlreadySet() throws Exception {
         assertWhileRunning(() -> {
-
-            Shell.Result result = installation.executables().setupPasswordsTool.run("auto --batch", null);
+            ServerUtils.waitForElasticsearch("green", null, installation, "elastic", BOOTSTRAP_PASSWORD, null);
+            Shell.Result result = retryOnAuthenticationErrors(
+                () -> installation.executables().setupPasswordsTool.run("auto --batch", null)
+            );
             Map<String, String> userpasses = parseUsersAndPasswords(result.stdout());
             assertThat(userpasses, hasKey("elastic"));
             for (Map.Entry<String, String> userpass : userpasses.entrySet()) {
@@ -130,6 +139,48 @@ public class PasswordToolsTests extends PackagingTestCase {
         });
     }
 
+    /**
+     * The security index is created on startup.
+     * It can happen that even when the security index exists, we get an authentication failure as `elastic`
+     * user because the reserved realm checks the security index first.
+     * This is because we check the security index too early (just after the creation) when all shards did not get allocated yet.
+     * Hence, the call can result in an `UnavailableShardsException` and cause the authentication to fail.
+     * We retry here on authentication errors for a couple of seconds just to verify that this is not the case.
+     */
+    private <R> R retryOnAuthenticationErrors(final Callable<R> callable) throws Exception {
+        Exception failure = null;
+        int retries = 5;
+        while (retries-- > 0) {
+            try {
+                return callable.call();
+            } catch (Exception e) {
+                if (e.getMessage() != null
+                    && (e.getMessage().contains("401 Unauthorized") || e.getMessage().contains("Failed to authenticate user"))) {
+                    logger.info(
+                        "Authentication failed (possibly due to UnavailableShardsException for the security index), retrying [{}].",
+                        retries,
+                        e
+                    );
+                    if (failure == null) {
+                        failure = e;
+                    } else {
+                        failure.addSuppressed(e);
+                    }
+                    try {
+                        Thread.sleep(1000);
+                    } catch (InterruptedException interrupted) {
+                        Thread.currentThread().interrupt();
+                        failure.addSuppressed(interrupted);
+                        throw failure;
+                    }
+                } else {
+                    throw e;
+                }
+            }
+        }
+        throw failure;
+    }
+
     private Map<String, String> parseUsersAndPasswords(String output) {
         Matcher matcher = USERPASS_REGEX.matcher(output);
         assertNotNull(matcher);

qa/packaging/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java

@@ -66,7 +66,7 @@ public class ServerUtils {
     private static final long waitTime = TimeUnit.MINUTES.toMillis(3);
     private static final long timeoutLength = TimeUnit.SECONDS.toMillis(30);
     private static final long requestInterval = TimeUnit.SECONDS.toMillis(5);
-    private static final long dockerWaitForSecurityIndex = TimeUnit.SECONDS.toMillis(25);
+    private static final long dockerWaitForSecurityIndex = TimeUnit.SECONDS.toMillis(60);
 
     public static void waitForElasticsearch(Installation installation) throws Exception {
         final boolean securityEnabled;
@@ -260,9 +260,7 @@ public class ServerUtils {
                         // `elastic` , the reserved realm checks the security index first. It can happen that we check the security index
                         // too early after the security index creation in DockerTests causing an UnavailableShardsException. We retry
                         // authentication errors for a couple of seconds just to verify this is not the case.
-                        if (installation.distribution.isDocker()
-                            && timeElapsed < dockerWaitForSecurityIndex
-                            && response.getStatusLine().getStatusCode() == 401) {
+                        if (timeElapsed < dockerWaitForSecurityIndex && response.getStatusLine().getStatusCode() == 401) {
                             logger.info(
                                 "Authentication against docker failed (possibly due to UnavailableShardsException for the security index)"
                                     + ", retrying..."

test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java

@@ -18,6 +18,7 @@ import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.store.AlreadyClosedException;
+import org.elasticsearch.action.UnavailableShardsException;
 import org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsRequest;
 import org.elasticsearch.action.admin.cluster.configuration.ClearVotingConfigExclusionsRequest;
 import org.elasticsearch.action.admin.cluster.configuration.TransportAddVotingConfigExclusionsAction;
@@ -145,6 +146,8 @@ import static org.elasticsearch.discovery.FileBasedSeedHostsProvider.UNICAST_HOS
 import static org.elasticsearch.node.Node.INITIAL_STATE_TIMEOUT_SETTING;
 import static org.elasticsearch.test.ESTestCase.TEST_REQUEST_TIMEOUT;
 import static org.elasticsearch.test.ESTestCase.assertBusy;
+import static org.elasticsearch.test.ESTestCase.assertFalse;
+import static org.elasticsearch.test.ESTestCase.assertTrue;
 import static org.elasticsearch.test.ESTestCase.randomFrom;
 import static org.elasticsearch.test.ESTestCase.runInParallel;
 import static org.elasticsearch.test.ESTestCase.safeAwait;
@@ -159,9 +162,7 @@ import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.not;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 /**
@@ -1239,16 +1240,29 @@ public final class InternalTestCluster extends TestCluster {
         }
         logger.trace("validating cluster formed, expecting {}", expectedNodes);
 
-        assertFalse(
-            client().admin()
-                .cluster()
-                .prepareHealth(TEST_REQUEST_TIMEOUT)
-                .setWaitForEvents(Priority.LANGUID)
-                .setWaitForNodes(Integer.toString(expectedNodes.size()))
-                .get(TimeValue.timeValueSeconds(40))
-                .isTimedOut()
-        );
         try {
+            assertBusy(() -> {
+                try {
+                    final boolean timeout = client().admin()
+                        .cluster()
+                        .prepareHealth(TEST_REQUEST_TIMEOUT)
+                        .setWaitForEvents(Priority.LANGUID)
+                        .setWaitForNodes(Integer.toString(expectedNodes.size()))
+                        .get(TimeValue.timeValueSeconds(40))
+                        .isTimedOut();
+                    if (timeout) {
+                        throw new IllegalStateException("timed out waiting for cluster to form");
+                    }
+                } catch (UnavailableShardsException e) {
+                    if (e.getMessage() != null && e.getMessage().contains(".security")) {
+                        // security index may not be ready yet, throwing assertion error to retry
+                        throw new AssertionError(e);
+                    } else {
+                        throw e;
+                    }
+                }
+            }, 30, TimeUnit.SECONDS);
+
             assertBusy(() -> {
                 final List<ClusterState> states = nodes.values()
                     .stream()

x-pack/plugin/build.gradle

@@ -219,4 +219,3 @@ tasks.named("yamlRestTestV7CompatTransform").configure({ task ->
   task.skipTest("esql/190_lookup_join/alias-pattern-single", "LOOKUP JOIN does not support index aliases for now")
 
 })
-

x-pack/plugin/core/build.gradle

@@ -153,6 +153,7 @@ testClusters.configureEach {
   keystore 'bootstrap.password', 'x-pack-test-password'
   user username: "x_pack_rest_user", password: "x-pack-test-password"
   requiresFeature 'es.failure_store_feature_flag_enabled', Version.fromString("8.15.0")
+  systemProperty 'es.queryable_built_in_roles_enabled', 'false'
 }
 
 if (buildParams.isSnapshotBuild() == false) {

x-pack/plugin/fleet/build.gradle

@@ -29,4 +29,5 @@ testClusters.configureEach {
   setting 'xpack.security.enabled', 'true'
   setting 'xpack.security.autoconfiguration.enabled', 'false'
   user username: 'x_pack_rest_user', password: 'x-pack-test-password'
+  systemProperty 'es.queryable_built_in_roles_enabled', 'false'
 }

x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetDataStreamIT.java

@@ -48,6 +48,11 @@ public class FleetDataStreamIT extends ESRestTestCase {
             .build();
     }
 
+    @Override
+    protected boolean preserveSecurityIndicesUponCompletion() {
+        return true;
+    }
+
     public void testAliasWithSystemDataStream() throws Exception {
         // Create a system data stream
         Request initialDocResponse = new Request("POST", ".fleet-actions-results/_doc");

x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecuritySpecialUserIT.java

@@ -218,7 +218,7 @@ public class RemoteClusterSecuritySpecialUserIT extends AbstractRemoteClusterSec
                 { "password": "%s" }""", PASS));
             assertOK(client().performRequest(changePasswordRequest));
 
-            final Request elasticUserSearchRequest = new Request("GET", "/*:.security*/_search");
+            final Request elasticUserSearchRequest = new Request("GET", "/*:.security*/_search?size=1");
             elasticUserSearchRequest.setOptions(
                 RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", basicAuthHeaderValue("elastic", PASS))
             );

x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/LicenseDLSFLSRoleIT.java

@@ -132,7 +132,8 @@ public final class LicenseDLSFLSRoleIT extends ESRestTestCase {
                     .build() };
             createRoleWithIndicesPrivileges(adminClient(), "role_with_FLS_and_DLS", indicesPrivileges);
         }
-        assertQuery(client(), "", 4, roles -> {
+        assertQuery(client(), """
+            {"query":{"bool":{"must_not":{"term":{"metadata._reserved":true}}}}}""", 4, roles -> {
             roles.sort(Comparator.comparing(o -> ((String) o.get("name"))));
             assertThat(roles, iterableWithSize(4));
             assertThat(roles.get(0).get("name"), equalTo("role_with_DLS"));
@@ -152,7 +153,8 @@ public final class LicenseDLSFLSRoleIT extends ESRestTestCase {
         assertTrue(((Boolean) responseMap.get("basic_was_started")));
         assertTrue(((Boolean) responseMap.get("acknowledged")));
         // now the same roles show up as disabled ("enabled" is "false")
-        assertQuery(client(), "", 4, roles -> {
+        assertQuery(client(), """
+            {"query":{"bool":{"must_not":{"term":{"metadata._reserved":true}}}}}""", 4, roles -> {
             roles.sort(Comparator.comparing(o -> ((String) o.get("name"))));
             assertThat(roles, iterableWithSize(4));
             assertThat(roles.get(0).get("name"), equalTo("role_with_DLS"));

x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryRoleIT.java

@@ -16,8 +16,10 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.ApplicationResourcePrivileges;
+import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
 import org.elasticsearch.xpack.security.support.SecurityMigrations;
 import org.hamcrest.Matchers;
+import org.junit.Before;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -49,15 +51,23 @@ public final class QueryRoleIT extends SecurityInBasicRestTestCase {
 
     private static final String READ_SECURITY_USER_AUTH_HEADER = "Basic cmVhZF9zZWN1cml0eV91c2VyOnJlYWQtc2VjdXJpdHktcGFzc3dvcmQ=";
 
-    public void testSimpleQueryAllRoles() throws IOException {
-        assertQuery("", 0, roles -> assertThat(roles, emptyIterable()));
-        RoleDescriptor createdRole = createRandomRole();
-        assertQuery("", 1, roles -> {
-            assertThat(roles, iterableWithSize(1));
-            assertRoleMap(roles.get(0), createdRole);
+    @Before
+    public void initialize() {
+        new ReservedRolesStore();
+    }
+
+    public void testSimpleQueryAllRoles() throws Exception {
+        createRandomRole();
+        assertQuery("", 1 + ReservedRolesStore.names().size(), roles -> {
+            // default size is 10
+            assertThat(roles, iterableWithSize(10));
         });
-        assertQuery("""
-            {"query":{"match_all":{}},"from":1}""", 1, roles -> assertThat(roles, emptyIterable()));
+        assertQuery(
+            Strings.format("""
+                {"query":{"match_all":{}},"from":%d}""", 1 + ReservedRolesStore.names().size()),
+            1 + ReservedRolesStore.names().size(),
+            roles -> assertThat(roles, emptyIterable())
+        );
     }
 
     public void testDisallowedFields() throws Exception {

x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/PermissionPrecedenceTests.java

@@ -16,6 +16,7 @@ import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.test.SecurityIntegTestCase;
 import org.elasticsearch.test.SecuritySettingsSourceField;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
+import org.junit.After;
 
 import java.util.Collections;
 import java.util.List;
@@ -35,9 +36,14 @@ import static org.hamcrest.Matchers.hasSize;
  */
 public class PermissionPrecedenceTests extends SecurityIntegTestCase {
 
+    @After
+    public void cleanupSecurityIndex() {
+        super.deleteSecurityIndex();
+    }
+
     @Override
     protected String configRoles() {
-        return """
+        return super.configRoles() + "\n" + """
             admin:
               cluster: [ all ]\s
               indices:
@@ -54,12 +60,22 @@ public class PermissionPrecedenceTests extends SecurityIntegTestCase {
         final String usersPasswdHashed = new String(
             getFastStoredHashAlgoForTests().hash(SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)
         );
-        return "admin:" + usersPasswdHashed + "\n" + "client:" + usersPasswdHashed + "\n" + "user:" + usersPasswdHashed + "\n";
+        return super.configUsers()
+            + "\n"
+            + "admin:"
+            + usersPasswdHashed
+            + "\n"
+            + "client:"
+            + usersPasswdHashed
+            + "\n"
+            + "user:"
+            + usersPasswdHashed
+            + "\n";
     }
 
     @Override
     protected String configUsersRoles() {
-        return """
+        return super.configUsersRoles() + "\n" + """
             admin:admin
             transport_client:client
             user:user

x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmElasticAutoconfigIntegTests.java

@@ -72,7 +72,7 @@ public class ReservedRealmElasticAutoconfigIntegTests extends SecuritySingleNode
 
     private boolean isMigrationComplete(ClusterState state) {
         IndexMetadata indexMetadata = state.metadata().getIndices().get(TestRestrictedIndices.INTERNAL_SECURITY_MAIN_INDEX_7);
-        return indexMetadata.getCustomData(MIGRATION_VERSION_CUSTOM_KEY) != null;
+        return indexMetadata != null && indexMetadata.getCustomData(MIGRATION_VERSION_CUSTOM_KEY) != null;
     }
 
     private void awaitSecurityMigrationRanOnce() {
@@ -89,8 +89,27 @@ public class ReservedRealmElasticAutoconfigIntegTests extends SecuritySingleNode
         safeAwait(latch);
     }
 
-    public void testAutoconfigFailedPasswordPromotion() {
+    private void deleteSecurityIndex() {
+        // delete the security index, if it exist
+        GetIndexRequest getIndexRequest = new GetIndexRequest();
+        getIndexRequest.indices(SECURITY_MAIN_ALIAS);
+        getIndexRequest.indicesOptions(IndicesOptions.lenientExpandOpen());
+        GetIndexResponse getIndexResponse = client().admin().indices().getIndex(getIndexRequest).actionGet();
+        if (getIndexResponse.getIndices().length > 0) {
+            assertThat(getIndexResponse.getIndices().length, is(1));
+            assertThat(getIndexResponse.getIndices()[0], is(TestRestrictedIndices.INTERNAL_SECURITY_MAIN_INDEX_7));
+
+            // Security migration needs to finish before deleting the index
+            awaitSecurityMigrationRanOnce();
+            DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(getIndexResponse.getIndices());
+            assertAcked(client().admin().indices().delete(deleteIndexRequest).actionGet());
+        }
+    }
+
+    public void testAutoconfigFailedPasswordPromotion() throws Exception {
         try {
+            // .security index is created automatically on node startup so delete the security index first
+            deleteSecurityIndex();
             // prevents the .security index from being created automatically (after elastic user authentication)
             ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(
                 TEST_REQUEST_TIMEOUT,
@@ -99,20 +118,6 @@ public class ReservedRealmElasticAutoconfigIntegTests extends SecuritySingleNode
             updateSettingsRequest.transientSettings(Settings.builder().put(Metadata.SETTING_READ_ONLY_ALLOW_DELETE_SETTING.getKey(), true));
             assertAcked(clusterAdmin().updateSettings(updateSettingsRequest).actionGet());
 
-            // delete the security index, if it exist
-            GetIndexRequest getIndexRequest = new GetIndexRequest();
-            getIndexRequest.indices(SECURITY_MAIN_ALIAS);
-            getIndexRequest.indicesOptions(IndicesOptions.lenientExpandOpen());
-            GetIndexResponse getIndexResponse = client().admin().indices().getIndex(getIndexRequest).actionGet();
-            if (getIndexResponse.getIndices().length > 0) {
-                assertThat(getIndexResponse.getIndices().length, is(1));
-                assertThat(getIndexResponse.getIndices()[0], is(TestRestrictedIndices.INTERNAL_SECURITY_MAIN_INDEX_7));
-                // Security migration needs to finish before deleting the index
-                awaitSecurityMigrationRanOnce();
-                DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(getIndexResponse.getIndices());
-                assertAcked(client().admin().indices().delete(deleteIndexRequest).actionGet());
-            }
-
             // elastic user gets 503 for the good password
             Request restRequest = randomFrom(
                 new Request("GET", "/_security/_authenticate"),

x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/ReadActionsTests.java

@@ -23,6 +23,7 @@ import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.test.SecurityIntegTestCase;
 import org.elasticsearch.test.SecuritySettingsSource;
+import org.junit.After;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -38,9 +39,14 @@ import static org.hamcrest.number.OrderingComparison.greaterThan;
 
 public class ReadActionsTests extends SecurityIntegTestCase {
 
+    @After
+    public void cleanupSecurityIndex() {
+        super.deleteSecurityIndex();
+    }
+
     @Override
     protected String configRoles() {
-        return Strings.format("""
+        return super.configRoles() + "\n" + Strings.format("""
             %s:
               cluster: [ ALL ]
               indices:

x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/WriteActionsTests.java

@@ -21,6 +21,7 @@ import org.elasticsearch.index.engine.DocumentMissingException;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.SecurityIntegTestCase;
 import org.elasticsearch.test.SecuritySettingsSource;
+import org.junit.After;
 
 import static org.elasticsearch.test.SecurityTestsUtils.assertAuthorizationExceptionDefaultUsers;
 import static org.elasticsearch.test.SecurityTestsUtils.assertThrowsAuthorizationExceptionDefaultUsers;
@@ -32,9 +33,14 @@ import static org.hamcrest.core.IsInstanceOf.instanceOf;
 
 public class WriteActionsTests extends SecurityIntegTestCase {
 
+    @After
+    public void cleanupSecurityIndex() {
+        super.deleteSecurityIndex();
+    }
+
     @Override
     protected String configRoles() {
-        return Strings.format("""
+        return super.configRoles() + "\n" + Strings.format("""
             %s:
               cluster: [ ALL ]
               indices:

x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/AbstractProfileIntegTestCase.java

@@ -49,7 +49,7 @@ public abstract class AbstractProfileIntegTestCase extends SecurityIntegTestCase
     }
 
     @Before
-    public void createNativeUsers() {
+    public void createNativeUsers() throws Exception {
         final PutUserRequest putUserRequest1 = new PutUserRequest();
         putUserRequest1.username(RAC_USER_NAME);
         putUserRequest1.roles(RAC_ROLE, NATIVE_RAC_ROLE);
@@ -57,6 +57,7 @@ public abstract class AbstractProfileIntegTestCase extends SecurityIntegTestCase
         putUserRequest1.passwordHash(nativeRacUserPasswordHash.toCharArray());
         putUserRequest1.email(RAC_USER_NAME + "@example.com");
         assertThat(client().execute(PutUserAction.INSTANCE, putUserRequest1).actionGet().created(), is(true));
+        assertSecurityIndexActive();
     }
 
     @Override

x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/support/CleanupRoleMappingDuplicatesMigrationIT.java

@@ -305,6 +305,8 @@ public class CleanupRoleMappingDuplicatesMigrationIT extends SecurityIntegTestCa
         internalCluster().setBootstrapMasterNodeIndex(0);
         final String masterNode = internalCluster().getMasterName();
         ensureGreen();
+        deleteSecurityIndex(); // hack to force a new security index to be created
+        ensureGreen();
         CountDownLatch awaitMigrations = awaitMigrationVersionUpdates(
             masterNode,
             SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION

x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/transport/filter/IpFilteringIntegrationTests.java

@@ -16,6 +16,8 @@ import org.elasticsearch.test.ESIntegTestCase.Scope;
 import org.elasticsearch.test.SecurityIntegTestCase;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.xpack.core.common.socket.SocketAccess;
+import org.junit.After;
+import org.junit.Before;
 import org.junit.BeforeClass;
 
 import java.io.IOException;
@@ -57,6 +59,16 @@ public class IpFilteringIntegrationTests extends SecurityIntegTestCase {
             .build();
     }
 
+    @Before
+    public void waitForSecurityIndex() throws Exception {
+        assertSecurityIndexActive();
+    }
+
+    @After
+    public void cleanupSecurityIndex() throws Exception {
+        super.deleteSecurityIndex();
+    }
+
     public void testThatIpFilteringIsIntegratedIntoNettyPipelineViaHttp() throws Exception {
         TransportAddress transportAddress = randomFrom(
             internalCluster().getDataNodeInstance(HttpServerTransport.class).boundAddress().boundAddresses()

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/QueryableBuiltInRolesSynchronizer.java

@@ -21,6 +21,7 @@ import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.ClusterStateTaskListener;
 import org.elasticsearch.cluster.NotMasterException;
 import org.elasticsearch.cluster.SimpleBatchedExecutor;
+import org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -87,9 +88,9 @@ public final class QueryableBuiltInRolesSynchronizer implements ClusterStateList
     public static final boolean QUERYABLE_BUILT_IN_ROLES_ENABLED;
     static {
         final var propertyValue = System.getProperty("es.queryable_built_in_roles_enabled");
-        if (propertyValue == null || propertyValue.isEmpty() || "false".equals(propertyValue)) {
+        if ("false".equals(propertyValue)) {
             QUERYABLE_BUILT_IN_ROLES_ENABLED = false;
-        } else if ("true".equals(propertyValue)) {
+        } else if (propertyValue == null || propertyValue.isEmpty() || "true".equals(propertyValue)) {
             QUERYABLE_BUILT_IN_ROLES_ENABLED = true;
         } else {
             throw new IllegalStateException(
@@ -309,7 +310,8 @@ public final class QueryableBuiltInRolesSynchronizer implements ClusterStateList
             || cause instanceof ResourceAlreadyExistsException
             || cause instanceof VersionConflictEngineException
             || cause instanceof DocumentMissingException
-            || cause instanceof FailedToMarkBuiltInRolesAsSyncedException;
+            || cause instanceof FailedToMarkBuiltInRolesAsSyncedException
+            || (e instanceof FailedToCommitClusterStateException && "node closed".equals(cause.getMessage()));
     }
 
     private static boolean isMixedVersionCluster(DiscoveryNodes nodes) {

x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java

@@ -6,6 +6,7 @@
  */
 package org.elasticsearch.test;
 
+import org.elasticsearch.ResourceAlreadyExistsException;
 import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
 import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
 import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules;
@@ -439,7 +440,11 @@ public abstract class SecurityIntegTestCase extends ESIntegTestCase {
         );
         CreateIndexRequest createIndexRequest = new CreateIndexRequest(SECURITY_MAIN_ALIAS).waitForActiveShards(ActiveShardCount.ALL)
             .masterNodeTimeout(TEST_REQUEST_TIMEOUT);
-        client.admin().indices().create(createIndexRequest).actionGet();
+        try {
+            client.admin().indices().create(createIndexRequest).actionGet();
+        } catch (ResourceAlreadyExistsException e) {
+            logger.info("Security index already exists, ignoring.", e);
+        }
     }
 
     protected static Index resolveSecurityIndex(Metadata metadata) {

x-pack/plugin/src/yamlRestTest/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java

@@ -17,6 +17,8 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.junit.Before;
 import org.junit.ClassRule;
 
+import java.util.Objects;
+
 public class XPackRestIT extends AbstractXPackRestTest {
 
     @ClassRule
@@ -47,6 +49,10 @@ public class XPackRestIT extends AbstractXPackRestTest {
         .configFile("testnode.pem", Resource.fromClasspath("org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem"))
         .configFile("testnode.crt", Resource.fromClasspath("org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt"))
         .configFile("service_tokens", Resource.fromClasspath("service_tokens"))
+        .systemProperty("es.queryable_built_in_roles_enabled", () -> {
+            final String enabled = System.getProperty("es.queryable_built_in_roles_enabled");
+            return Objects.requireNonNullElse(enabled, "");
+        })
         .build();
 
     public XPackRestIT(ClientYamlTestCandidate testCandidate) {

x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/roles/50_remote_only.yml

@@ -75,7 +75,17 @@ teardown:
   - do:
       security.query_role:
         body: >
-          {}
+          {
+            "query": {
+              "bool": {
+                "must_not": {
+                  "term": {
+                    "metadata._reserved": true
+                  }
+                }
+              }
+            }
+          }
   - match: { total: 1 }
   - match: { count: 1 }
   - match: { roles.0.name: "remote_role" }

x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/roles/60_bulk_roles.yml

@@ -81,7 +81,16 @@ teardown:
       security.query_role:
         body: >
           {
-            "query": { "match_all": {} }, "sort": ["name"]
+            "query": {
+              "bool": {
+                "must_not": {
+                  "term": {
+                    "metadata._reserved": true
+                  }
+                }
+              }
+            },
+          "sort": ["name"]
           }
   - match: { total: 2 }
   - match: { count: 2 }