Tweak security system indexes refresh behavior for stateless (#98285)

This is an omnibus PR that:
 * toggles the fast_refresh option for the profiles index (to make it alike to the .kibana index)
 * makes API Key creation refresh policy default to IMMEDIATE from WAIT_UNTIL in serverless, to mitigate the long automatic refresh interval

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/AbstractCreateApiKeyRequest.java

@@ -21,17 +21,17 @@ import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 
 public abstract class AbstractCreateApiKeyRequest extends ActionRequest {
-    public static final WriteRequest.RefreshPolicy DEFAULT_REFRESH_POLICY = WriteRequest.RefreshPolicy.WAIT_UNTIL;
     protected final String id;
     protected String name;
     protected TimeValue expiration;
     protected Map<String, Object> metadata;
     protected List<RoleDescriptor> roleDescriptors = Collections.emptyList();
-    protected WriteRequest.RefreshPolicy refreshPolicy = DEFAULT_REFRESH_POLICY;
+    protected WriteRequest.RefreshPolicy refreshPolicy;
 
     public AbstractCreateApiKeyRequest() {
         super();
@@ -68,6 +68,10 @@ public abstract class AbstractCreateApiKeyRequest extends ActionRequest {
         return refreshPolicy;
     }
 
+    public void setRefreshPolicy(WriteRequest.RefreshPolicy refreshPolicy) {
+        this.refreshPolicy = Objects.requireNonNull(refreshPolicy, "refresh policy may not be null");
+    }
+
     public Map<String, Object> getMetadata() {
         return metadata;
     }
@@ -94,6 +98,7 @@ public abstract class AbstractCreateApiKeyRequest extends ActionRequest {
                 validationException
             );
         }
+        assert refreshPolicy != null : "refresh policy is required";
         return validationException;
     }
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/CreateApiKeyRequest.java

@@ -21,7 +21,6 @@ import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 
 /**
  * Request class used for the creation of an API key. The request requires a name to be provided
@@ -103,10 +102,6 @@ public final class CreateApiKeyRequest extends AbstractCreateApiKeyRequest {
         this.roleDescriptors = (roleDescriptors == null) ? List.of() : List.copyOf(roleDescriptors);
     }
 
-    public void setRefreshPolicy(WriteRequest.RefreshPolicy refreshPolicy) {
-        this.refreshPolicy = Objects.requireNonNull(refreshPolicy, "refresh policy may not be null");
-    }
-
     public void setMetadata(Map<String, Object> metadata) {
         this.metadata = metadata;
     }

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/CreateApiKeyRequestTests.java

@@ -21,6 +21,9 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.NONE;
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.WAIT_UNTIL;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.containsStringIgnoringCase;
 import static org.hamcrest.Matchers.equalTo;
@@ -31,6 +34,7 @@ public class CreateApiKeyRequestTests extends ESTestCase {
     public void testNameValidation() {
         final String name = randomAlphaOfLengthBetween(1, 256);
         CreateApiKeyRequest request = new CreateApiKeyRequest();
+        request.setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL, NONE));
 
         ActionRequestValidationException ve = request.validate();
         assertThat(ve.validationErrors().size(), is(1));
@@ -78,6 +82,7 @@ public class CreateApiKeyRequestTests extends ESTestCase {
     public void testMetadataKeyValidation() {
         final String name = randomAlphaOfLengthBetween(1, 256);
         CreateApiKeyRequest request = new CreateApiKeyRequest();
+        request.setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL, NONE));
         request.setName(name);
         request.setMetadata(Map.of("_foo", "bar"));
         final ActionRequestValidationException ve = request.validate();
@@ -112,6 +117,7 @@ public class CreateApiKeyRequestTests extends ESTestCase {
             ),
             null
         );
+        request1.setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL, NONE));
         final ActionRequestValidationException ve1 = request1.validate();
         assertNotNull(ve1);
         assertThat(ve1.validationErrors().get(0), containsString("unknown cluster privilege"));

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/CreateCrossClusterApiKeyRequestTests.java

@@ -17,6 +17,10 @@ import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.NONE;
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.WAIT_UNTIL;
+
 public class CreateCrossClusterApiKeyRequestTests extends AbstractWireSerializingTestCase<CreateCrossClusterApiKeyRequest> {
 
     private String access;
@@ -35,12 +39,14 @@ public class CreateCrossClusterApiKeyRequestTests extends AbstractWireSerializin
 
     @Override
     protected CreateCrossClusterApiKeyRequest createTestInstance() {
-        return new CreateCrossClusterApiKeyRequest(
+        CreateCrossClusterApiKeyRequest request = new CreateCrossClusterApiKeyRequest(
             randomAlphaOfLengthBetween(3, 8),
             roleDescriptorBuilder,
             randomExpiration(),
             randomMetadata()
         );
+        request.setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL, NONE));
+        return request;
     }
 
     @Override

x-pack/plugin/identity-provider/src/internalClusterTest/java/org/elasticsearch/xpack/idp/action/SamlIdentityProviderTests.java

@@ -8,7 +8,6 @@
 package org.elasticsearch.xpack.idp.action;
 
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
@@ -57,6 +56,9 @@ import java.util.concurrent.TimeUnit;
 import java.util.zip.Deflater;
 import java.util.zip.DeflaterOutputStream;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.NONE;
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.WAIT_UNTIL;
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
 import static org.hamcrest.Matchers.containsString;
@@ -405,10 +407,7 @@ public class SamlIdentityProviderTests extends IdentityProviderIntegTestCase {
             )
         );
         spFields.put("privileges", Map.of("resource", entityId, "roles", Set.of("sso:(\\w+)")));
-        Request request = new Request(
-            "PUT",
-            "/_idp/saml/sp/" + urlEncode(entityId) + "?refresh=" + WriteRequest.RefreshPolicy.IMMEDIATE.getValue()
-        );
+        Request request = new Request("PUT", "/_idp/saml/sp/" + urlEncode(entityId) + "?refresh=" + IMMEDIATE.getValue());
         request.setOptions(REQUEST_OPTIONS_AS_CONSOLE_USER);
         final XContentBuilder builder = XContentFactory.jsonBuilder();
         builder.map(spFields);
@@ -437,7 +436,7 @@ public class SamlIdentityProviderTests extends IdentityProviderIntegTestCase {
     }
 
     private void registerApplicationPrivileges(Map<String, Set<String>> privileges) throws IOException {
-        Request request = new Request("PUT", "/_security/privilege?refresh=" + WriteRequest.RefreshPolicy.IMMEDIATE.getValue());
+        Request request = new Request("PUT", "/_security/privilege?refresh=" + IMMEDIATE.getValue());
         request.setOptions(REQUEST_OPTIONS_AS_CONSOLE_USER);
         final XContentBuilder builder = XContentFactory.jsonBuilder();
         builder.startObject();
@@ -487,6 +486,7 @@ public class SamlIdentityProviderTests extends IdentityProviderIntegTestCase {
         );
         final CreateApiKeyResponse response = new CreateApiKeyRequestBuilder(client).setName("test key")
             .setExpiration(TimeValue.timeValueHours(TimeUnit.DAYS.toHours(7L)))
+            .setRefreshPolicy(randomFrom(WAIT_UNTIL, IMMEDIATE, NONE))
             .get();
         assertNotNull(response);
         return Base64.getEncoder().encodeToString((response.getId() + ":" + response.getKey().toString()).getBytes(StandardCharsets.UTF_8));

x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DlsFlsRequestCacheTests.java

@@ -42,6 +42,9 @@ import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.NONE;
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.WAIT_UNTIL;
 import static org.elasticsearch.test.SecuritySettingsSource.TEST_PASSWORD_HASHED;
 import static org.elasticsearch.test.SecuritySettingsSourceField.TEST_PASSWORD;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -455,6 +458,7 @@ public class DlsFlsRequestCacheTests extends SecuritySingleNodeTestCase {
             ),
             null
         );
+        createApiKeyRequest.setRefreshPolicy(randomFrom(WAIT_UNTIL, IMMEDIATE, NONE));
         final CreateApiKeyResponse createApiKeyResponse = limitedClient().execute(CreateApiKeyAction.INSTANCE, createApiKeyRequest).get();
 
         final String base64ApiKey = Base64.getEncoder()

x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java

@@ -20,7 +20,6 @@ import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
 import org.elasticsearch.action.get.GetAction;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.support.PlainActionFuture;
-import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.action.update.UpdateResponse;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.RequestOptions;
@@ -125,6 +124,9 @@ import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.NONE;
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.WAIT_UNTIL;
 import static org.elasticsearch.test.SecuritySettingsSource.ES_TEST_ROOT_USER;
 import static org.elasticsearch.test.SecuritySettingsSource.HASHER;
 import static org.elasticsearch.test.SecuritySettingsSource.TEST_ROLE;
@@ -264,6 +266,7 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
             .setExpiration(TimeValue.timeValueHours(TimeUnit.DAYS.toHours(7L)))
             .setRoleDescriptors(Collections.singletonList(descriptor))
             .setMetadata(ApiKeyTests.randomMetadata())
+            .setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL))
             .get();
 
         assertEquals("test key", response.getName());
@@ -278,7 +281,9 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
         assertThat(getApiKeyInfo(client(), response.getId(), randomBoolean(), randomBoolean()).getType(), is(ApiKey.Type.REST));
 
         // create simple api key
-        final CreateApiKeyResponse simple = new CreateApiKeyRequestBuilder(client).setName("simple").get();
+        final CreateApiKeyResponse simple = new CreateApiKeyRequestBuilder(client).setName("simple")
+            .setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL))
+            .get();
         assertEquals("simple", simple.getName());
         assertNotNull(simple.getId());
         assertNotNull(simple.getKey());
@@ -320,7 +325,7 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
                 .setExpiration(null)
                 .setRoleDescriptors(Collections.singletonList(descriptor))
                 .setMetadata(ApiKeyTests.randomMetadata())
-                .setRefreshPolicy(WriteRequest.RefreshPolicy.NONE)
+                .setRefreshPolicy(NONE)
                 .get();
             assertNotNull(response.getId());
             assertNotNull(response.getKey());
@@ -338,7 +343,7 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
         );
         final ActionRequestValidationException e = expectThrows(
             ActionRequestValidationException.class,
-            () -> new CreateApiKeyRequestBuilder(client).get()
+            () -> new CreateApiKeyRequestBuilder(client).setRefreshPolicy(randomFrom(NONE, WAIT_UNTIL, IMMEDIATE)).get()
         );
         assertThat(e.getMessage(), containsString("api key name is required"));
     }
@@ -643,7 +648,7 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
         assertFalse(created.isBefore(withinRetention));
         UpdateResponse expirationDateUpdatedResponse = client.prepareUpdate(SECURITY_MAIN_ALIAS, createdApiKeys.get(0).getId())
             .setDoc("expiration_time", withinRetention.toEpochMilli())
-            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+            .setRefreshPolicy(IMMEDIATE)
             .get();
         assertThat(expirationDateUpdatedResponse.getResult(), is(DocWriteResponse.Result.UPDATED));
 
@@ -653,21 +658,21 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
         assertTrue(Instant.now().isAfter(outsideRetention));
         expirationDateUpdatedResponse = client.prepareUpdate(SECURITY_MAIN_ALIAS, createdApiKeys.get(1).getId())
             .setDoc("expiration_time", outsideRetention.toEpochMilli())
-            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+            .setRefreshPolicy(IMMEDIATE)
             .get();
         assertThat(expirationDateUpdatedResponse.getResult(), is(DocWriteResponse.Result.UPDATED));
 
         // Invalidate the 3rd key such that it cannot be deleted by the remover
         UpdateResponse invalidateUpdateResponse = client.prepareUpdate(SECURITY_MAIN_ALIAS, createdApiKeys.get(2).getId())
             .setDoc("invalidation_time", withinRetention.toEpochMilli(), "api_key_invalidated", true)
-            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+            .setRefreshPolicy(IMMEDIATE)
             .get();
         assertThat(invalidateUpdateResponse.getResult(), is(DocWriteResponse.Result.UPDATED));
 
         // Invalidate the 4th key such that it will be deleted by the remover
         invalidateUpdateResponse = client.prepareUpdate(SECURITY_MAIN_ALIAS, createdApiKeys.get(3).getId())
             .setDoc("invalidation_time", outsideRetention.toEpochMilli(), "api_key_invalidated", true)
-            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+            .setRefreshPolicy(IMMEDIATE)
             .get();
         assertThat(invalidateUpdateResponse.getResult(), is(DocWriteResponse.Result.UPDATED));
 
@@ -681,7 +686,7 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
                 "api_key_invalidated",
                 true
             )
-            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+            .setRefreshPolicy(IMMEDIATE)
             .get();
         assertThat(updateResponse.getResult(), is(DocWriteResponse.Result.UPDATED));
 
@@ -695,7 +700,7 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
                 "api_key_invalidated",
                 true
             )
-            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+            .setRefreshPolicy(IMMEDIATE)
             .get();
         assertThat(updateResponse.getResult(), is(DocWriteResponse.Result.UPDATED));
 
@@ -703,7 +708,7 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
         // It does not matter whether it has an expiration time or whether the expiration time is still within retention period
         updateResponse = client.prepareUpdate(SECURITY_MAIN_ALIAS, createdApiKeys.get(6).getId())
             .setDoc("api_key_invalidated", true, "expiration_time", randomBoolean() ? withinRetention.toEpochMilli() : null)
-            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+            .setRefreshPolicy(IMMEDIATE)
             .get();
         assertThat(updateResponse.getResult(), is(DocWriteResponse.Result.UPDATED));
 
@@ -1599,6 +1604,7 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
                 Collections.singletonList(new RoleDescriptor("role", new String[] { "manage_api_key", "manage_token" }, null, null))
             )
             .setMetadata(ApiKeyTests.randomMetadata())
+            .setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL, NONE))
             .get();
 
         assertEquals("key-1", response.getName());
@@ -1624,13 +1630,19 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
 
         final IllegalArgumentException e1 = expectThrows(
             IllegalArgumentException.class,
-            () -> new CreateApiKeyRequestBuilder(clientKey1).setName("key-2").setMetadata(ApiKeyTests.randomMetadata()).get()
+            () -> new CreateApiKeyRequestBuilder(clientKey1).setName("key-2")
+                .setMetadata(ApiKeyTests.randomMetadata())
+                .setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL, NONE))
+                .get()
         );
         assertThat(e1.getMessage(), containsString(expectedMessage));
 
         final IllegalArgumentException e2 = expectThrows(
             IllegalArgumentException.class,
-            () -> new CreateApiKeyRequestBuilder(clientKey1).setName("key-3").setRoleDescriptors(Collections.emptyList()).get()
+            () -> new CreateApiKeyRequestBuilder(clientKey1).setName("key-3")
+                .setRoleDescriptors(Collections.emptyList())
+                .setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL, NONE))
+                .get()
         );
         assertThat(e2.getMessage(), containsString(expectedMessage));
 
@@ -1641,6 +1653,7 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
                 .setRoleDescriptors(
                     Collections.singletonList(new RoleDescriptor("role", new String[] { "manage_own_api_key" }, null, null))
                 )
+                .setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL, NONE))
                 .get()
         );
         assertThat(e3.getMessage(), containsString(expectedMessage));
@@ -1656,6 +1669,7 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
             () -> new CreateApiKeyRequestBuilder(clientKey1).setName("key-5")
                 .setMetadata(ApiKeyTests.randomMetadata())
                 .setRoleDescriptors(roleDescriptors)
+                .setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL, NONE))
                 .get()
         );
         assertThat(e4.getMessage(), containsString(expectedMessage));
@@ -1663,6 +1677,7 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
         final CreateApiKeyResponse key100Response = new CreateApiKeyRequestBuilder(clientKey1).setName("key-100")
             .setMetadata(ApiKeyTests.randomMetadata())
             .setRoleDescriptors(Collections.singletonList(new RoleDescriptor("role", null, null, null)))
+            .setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL))
             .get();
         assertEquals("key-100", key100Response.getName());
         assertNotNull(key100Response.getId());
@@ -1696,6 +1711,7 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
         final CreateApiKeyResponse response1 = new CreateApiKeyRequestBuilder(client).setName("run-as-key")
             .setRoleDescriptors(List.of(descriptor))
             .setMetadata(ApiKeyTests.randomMetadata())
+            .setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL))
             .get();
 
         final String base64ApiKeyKeyValue = Base64.getEncoder()
@@ -1705,7 +1721,10 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
             client().filterWithHeader(
                 Map.of("Authorization", "ApiKey " + base64ApiKeyKeyValue, "es-security-runas-user", ES_TEST_ROOT_USER)
             )
-        ).setName("create-by run-as user").setRoleDescriptors(List.of(new RoleDescriptor("a", new String[] { "all" }, null, null))).get();
+        ).setName("create-by run-as user")
+            .setRoleDescriptors(List.of(new RoleDescriptor("a", new String[] { "all" }, null, null)))
+            .setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL))
+            .get();
 
         final GetApiKeyResponse getApiKeyResponse = client.execute(
             GetApiKeyAction.INSTANCE,
@@ -1732,6 +1751,7 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
         final CreateApiKeyResponse createApiKeyResponse = new CreateApiKeyRequestBuilder(client).setName("auth only key")
             .setRoleDescriptors(Collections.singletonList(descriptor))
             .setMetadata(ApiKeyTests.randomMetadata())
+            .setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL, NONE))
             .get();
 
         assertNotNull(createApiKeyResponse.getId());
@@ -2344,7 +2364,7 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
             assertTrue(Instant.now().isAfter(dayBefore));
             final var expirationDateUpdatedResponse = client().prepareUpdate(SECURITY_MAIN_ALIAS, apiKeyId)
                 .setDoc("expiration_time", dayBefore.toEpochMilli())
-                .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+                .setRefreshPolicy(IMMEDIATE)
                 .get();
             assertThat(expirationDateUpdatedResponse.getResult(), is(DocWriteResponse.Result.UPDATED));
         }
@@ -2846,6 +2866,7 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
 
         final CreateApiKeyResponse createApiKeyResponse = new CreateApiKeyRequestBuilder(client).setName("test key")
             .setMetadata(ApiKeyTests.randomMetadata())
+            .setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL))
             .get();
         final String docId = createApiKeyResponse.getId();
         authenticateWithApiKey(docId, createApiKeyResponse.getKey());
@@ -3087,7 +3108,7 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
                 .setExpiration(expiration)
                 .setRoleDescriptors(Collections.singletonList(descriptor))
                 .setMetadata(metadata)
-                .setRefreshPolicy(i == noOfApiKeys - 1 ? WriteRequest.RefreshPolicy.IMMEDIATE : WriteRequest.RefreshPolicy.NONE)
+                .setRefreshPolicy(i == noOfApiKeys - 1 ? randomFrom(IMMEDIATE, WAIT_UNTIL) : NONE)
                 .get();
             assertNotNull(response.getId());
             assertNotNull(response.getKey());

x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java

@@ -531,6 +531,10 @@ public class TokenAuthIntegTests extends SecurityIntegTestCase {
         // We now have two documents, the original(now refreshed) token doc and the new one with the new access doc
         AtomicReference<String> docId = new AtomicReference<>();
         assertBusy(() -> {
+            // refresh to make sure the token docs are visible
+            Request refreshRequest = new Request(HttpPost.METHOD_NAME, SecuritySystemIndices.SECURITY_TOKENS_ALIAS + "/_refresh");
+            refreshRequest.setOptions(SECURITY_REQUEST_OPTIONS);
+            getRestClient().performRequest(refreshRequest);
             Request searchRequest = new Request(HttpPost.METHOD_NAME, SecuritySystemIndices.SECURITY_TOKENS_ALIAS + "/_search");
             searchRequest.setOptions(SECURITY_REQUEST_OPTIONS);
             searchRequest.setJsonEntity("""

x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/apikey/ApiKeySingleNodeTests.java

@@ -82,6 +82,9 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.NONE;
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.WAIT_UNTIL;
 import static org.elasticsearch.test.SecuritySettingsSource.ES_TEST_ROOT_USER;
 import static org.elasticsearch.test.SecuritySettingsSourceField.TEST_PASSWORD;
 import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
@@ -115,14 +118,12 @@ public class ApiKeySingleNodeTests extends SecuritySingleNodeTestCase {
     }
 
     public void testQueryWithExpiredKeys() throws InterruptedException {
-        final String id1 = client().execute(
-            CreateApiKeyAction.INSTANCE,
-            new CreateApiKeyRequest("expired-shortly", null, TimeValue.timeValueMillis(1), null)
-        ).actionGet().getId();
-        final String id2 = client().execute(
-            CreateApiKeyAction.INSTANCE,
-            new CreateApiKeyRequest("long-lived", null, TimeValue.timeValueDays(1), null)
-        ).actionGet().getId();
+        CreateApiKeyRequest request1 = new CreateApiKeyRequest("expired-shortly", null, TimeValue.timeValueMillis(1), null);
+        request1.setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL));
+        final String id1 = client().execute(CreateApiKeyAction.INSTANCE, request1).actionGet().getId();
+        CreateApiKeyRequest request2 = new CreateApiKeyRequest("long-lived", null, TimeValue.timeValueDays(1), null);
+        request2.setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL));
+        final String id2 = client().execute(CreateApiKeyAction.INSTANCE, request2).actionGet().getId();
         Thread.sleep(10); // just to be 100% sure that the 1st key is expired when we search for it
 
         final QueryApiKeyRequest queryApiKeyRequest = new QueryApiKeyRequest(
@@ -150,6 +151,7 @@ public class ApiKeySingleNodeTests extends SecuritySingleNodeTestCase {
         grantApiKeyRequest.getGrant().setType("password");
         grantApiKeyRequest.getGrant().setUsername(username);
         grantApiKeyRequest.getGrant().setPassword(password);
+        grantApiKeyRequest.setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL, NONE));
         grantApiKeyRequest.getApiKeyRequest().setName(randomAlphaOfLength(8));
         grantApiKeyRequest.getApiKeyRequest()
             .setRoleDescriptors(
@@ -213,9 +215,11 @@ public class ApiKeySingleNodeTests extends SecuritySingleNodeTestCase {
             createServiceAccountTokenRequest
         ).actionGet();
 
+        CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest(randomAlphaOfLength(8), null, null);
+        createApiKeyRequest.setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL, NONE));
         final CreateApiKeyResponse createApiKeyResponse = client().filterWithHeader(
             Map.of("Authorization", "Bearer " + createServiceAccountTokenResponse.getValue())
-        ).execute(CreateApiKeyAction.INSTANCE, new CreateApiKeyRequest(randomAlphaOfLength(8), null, null)).actionGet();
+        ).execute(CreateApiKeyAction.INSTANCE, createApiKeyRequest).actionGet();
 
         final Map<String, Object> apiKeyDocument = getApiKeyDocument(createApiKeyResponse.getId());
 
@@ -237,16 +241,14 @@ public class ApiKeySingleNodeTests extends SecuritySingleNodeTestCase {
     }
 
     public void testGetApiKeyWorksForTheApiKeyItself() {
-        final String apiKeyName = randomAlphaOfLength(10);
-        final CreateApiKeyResponse createApiKeyResponse = client().execute(
-            CreateApiKeyAction.INSTANCE,
-            new CreateApiKeyRequest(
-                apiKeyName,
-                List.of(new RoleDescriptor("x", new String[] { "manage_own_api_key", "manage_token" }, null, null, null, null, null, null)),
-                null,
-                null
-            )
-        ).actionGet();
+        CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest(
+            randomAlphaOfLength(10),
+            List.of(new RoleDescriptor("x", new String[] { "manage_own_api_key", "manage_token" }, null, null, null, null, null, null)),
+            null,
+            null
+        );
+        createApiKeyRequest.setRefreshPolicy(randomFrom(WAIT_UNTIL, IMMEDIATE));
+        final CreateApiKeyResponse createApiKeyResponse = client().execute(CreateApiKeyAction.INSTANCE, createApiKeyRequest).actionGet();
 
         final String apiKeyId = createApiKeyResponse.getId();
         final String base64ApiKeyKeyValue = Base64.getEncoder()
@@ -360,6 +362,7 @@ public class ApiKeySingleNodeTests extends SecuritySingleNodeTestCase {
         ).createTokenWithClientCredentialsGrant();
         final GrantApiKeyRequest grantApiKeyRequest3 = new GrantApiKeyRequest();
         grantApiKeyRequest3.getApiKeyRequest().setName("granted-api-key-must-not-have-chained-runas");
+        grantApiKeyRequest3.setRefreshPolicy(randomFrom(NONE, WAIT_UNTIL, IMMEDIATE));
         grantApiKeyRequest3.getGrant().setType("access_token");
         grantApiKeyRequest3.getGrant().setAccessToken(new SecureString(oAuth2Token3.accessToken().toCharArray()));
         grantApiKeyRequest3.getGrant().setRunAsUsername("user2");
@@ -383,6 +386,7 @@ public class ApiKeySingleNodeTests extends SecuritySingleNodeTestCase {
         securityClient.putRole(new RoleDescriptor("user1_role", new String[] { "manage_token" }, null, new String[] { "user2" }));
         final GrantApiKeyRequest grantApiKeyRequest4 = new GrantApiKeyRequest();
         grantApiKeyRequest4.getApiKeyRequest().setName("granted-api-key-will-check-token-run-as-privilege");
+        grantApiKeyRequest4.setRefreshPolicy(randomFrom(NONE, WAIT_UNTIL, IMMEDIATE));
         grantApiKeyRequest4.getGrant().setType("access_token");
         grantApiKeyRequest4.getGrant().setAccessToken(new SecureString(oAuth2Token4.accessToken().toCharArray()));
         final ElasticsearchStatusException e4 = expectThrows(
@@ -400,10 +404,14 @@ public class ApiKeySingleNodeTests extends SecuritySingleNodeTestCase {
     }
 
     public void testInvalidateApiKeyWillRecordTimestamp() {
-        final String apiKeyId = client().execute(
-            CreateApiKeyAction.INSTANCE,
-            new CreateApiKeyRequest(randomAlphaOfLengthBetween(3, 8), null, TimeValue.timeValueMillis(randomLongBetween(1, 1000)), null)
-        ).actionGet().getId();
+        CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest(
+            randomAlphaOfLengthBetween(3, 8),
+            null,
+            TimeValue.timeValueMillis(randomLongBetween(1, 1000)),
+            null
+        );
+        createApiKeyRequest.setRefreshPolicy(randomFrom(WAIT_UNTIL, IMMEDIATE));
+        final String apiKeyId = client().execute(CreateApiKeyAction.INSTANCE, createApiKeyRequest).actionGet().getId();
         assertThat(getApiKeyDocument(apiKeyId).get("invalidation_time"), nullValue());
 
         final long start = Instant.now().toEpochMilli();
@@ -433,6 +441,7 @@ public class ApiKeySingleNodeTests extends SecuritySingleNodeTestCase {
             {
               "search": [ {"names": ["logs"]} ]
             }""");
+        request.setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL));
 
         final PlainActionFuture<CreateApiKeyResponse> future = new PlainActionFuture<>();
         client().execute(CreateCrossClusterApiKeyAction.INSTANCE, request, future);
@@ -535,6 +544,7 @@ public class ApiKeySingleNodeTests extends SecuritySingleNodeTestCase {
             {
               "search": [ {"names": ["logs"]} ]
             }""");
+        createApiKeyRequest.setRefreshPolicy(randomFrom(WAIT_UNTIL, IMMEDIATE));
         final CreateApiKeyResponse createApiKeyResponse = client().execute(CreateCrossClusterApiKeyAction.INSTANCE, createApiKeyRequest)
             .actionGet();
         final String apiKeyId = createApiKeyResponse.getId();
@@ -631,6 +641,7 @@ public class ApiKeySingleNodeTests extends SecuritySingleNodeTestCase {
                     null
                 )
             )
+            .setRefreshPolicy(randomFrom(NONE, WAIT_UNTIL, IMMEDIATE))
             .execute()
             .actionGet();
         final String encoded = Base64.getEncoder()
@@ -642,6 +653,7 @@ public class ApiKeySingleNodeTests extends SecuritySingleNodeTestCase {
             {
               "search": [ {"names": ["logs"]} ]
             }""");
+        request.setRefreshPolicy(randomFrom(NONE, WAIT_UNTIL, IMMEDIATE));
 
         final PlainActionFuture<CreateApiKeyResponse> future = new PlainActionFuture<>();
         client().filterWithHeader(Map.of("Authorization", "ApiKey " + encoded))
@@ -659,6 +671,7 @@ public class ApiKeySingleNodeTests extends SecuritySingleNodeTestCase {
         final GrantApiKeyRequest grantApiKeyRequest = new GrantApiKeyRequest();
         // randomly use either password or access token grant
         grantApiKeyRequest.getApiKeyRequest().setName("granted-api-key-for-" + username + "-runas-" + runAsUsername);
+        grantApiKeyRequest.setRefreshPolicy(randomFrom(WAIT_UNTIL, IMMEDIATE));
         if (randomBoolean()) {
             grantApiKeyRequest.getApiKeyRequest()
                 .setRoleDescriptors(List.of(new RoleDescriptor(randomAlphaOfLengthBetween(3, 8), new String[] { "monitor" }, null, null)));

x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreSingleNodeTests.java

@@ -46,6 +46,9 @@ import java.util.Set;
 import java.util.stream.Collectors;
 
 import static java.util.Collections.emptyMap;
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.NONE;
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.WAIT_UNTIL;
 import static org.elasticsearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES;
 import static org.elasticsearch.test.SecuritySettingsSourceField.TEST_PASSWORD;
 import static org.elasticsearch.test.SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING;
@@ -139,6 +142,7 @@ public class NativePrivilegeStoreSingleNodeTests extends SecuritySingleNodeTestC
             if (randomBoolean()) {
                 final var createApiKeyRequest = new CreateApiKeyRequest();
                 createApiKeyRequest.setName(randomAlphaOfLength(5));
+                createApiKeyRequest.setRefreshPolicy(randomFrom(NONE, IMMEDIATE, WAIT_UNTIL));
                 if (randomBoolean()) {
                     createApiKeyRequest.setRoleDescriptors(
                         List.of(

x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/SecurityDomainIntegTests.java

@@ -43,6 +43,9 @@ import java.util.Base64;
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.NONE;
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.WAIT_UNTIL;
 import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
 import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_MAIN_ALIAS;
 import static org.hamcrest.Matchers.containsString;
@@ -309,6 +312,7 @@ public class SecurityDomainIntegTests extends AbstractProfileIntegTestCase {
 
     public void testDomainCaptureForApiKey() {
         final CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest(randomAlphaOfLengthBetween(3, 8), null, null);
+        createApiKeyRequest.setRefreshPolicy(randomFrom(NONE, WAIT_UNTIL, IMMEDIATE));
 
         final CreateApiKeyResponse createApiKeyResponse = client().filterWithHeader(
             Map.of("Authorization", basicAuthHeaderValue(RAC_USER_NAME, NATIVE_RAC_USER_PASSWORD.clone()))

x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/user/AnonymousUserIntegTests.java

@@ -39,6 +39,9 @@ import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.NONE;
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.WAIT_UNTIL;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItems;
@@ -107,10 +110,9 @@ public class AnonymousUserIntegTests extends SecurityIntegTestCase {
     }
 
     public void testAnonymousRoleShouldBeCaptureWhenCreatingApiKey() throws IOException {
-        final CreateApiKeyResponse createApiKeyResponse = client().execute(
-            CreateApiKeyAction.INSTANCE,
-            new CreateApiKeyRequest(randomAlphaOfLength(8), null, null)
-        ).actionGet();
+        CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest(randomAlphaOfLength(8), null, null);
+        createApiKeyRequest.setRefreshPolicy(randomFrom(NONE, WAIT_UNTIL, IMMEDIATE));
+        final CreateApiKeyResponse createApiKeyResponse = client().execute(CreateApiKeyAction.INSTANCE, createApiKeyRequest).actionGet();
 
         final Map<String, Object> apiKeyDocument = getApiKeyDocument(createApiKeyResponse.getId());
 
@@ -130,9 +132,11 @@ public class AnonymousUserIntegTests extends SecurityIntegTestCase {
             createServiceAccountTokenRequest
         ).actionGet();
 
+        CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest(randomAlphaOfLength(8), null, null);
+        createApiKeyRequest.setRefreshPolicy(randomFrom(NONE, IMMEDIATE, WAIT_UNTIL));
         final CreateApiKeyResponse createApiKeyResponse = client().filterWithHeader(
             Map.of("Authorization", "Bearer " + createServiceAccountTokenResponse.getValue())
-        ).execute(CreateApiKeyAction.INSTANCE, new CreateApiKeyRequest(randomAlphaOfLength(8), null, null)).actionGet();
+        ).execute(CreateApiKeyAction.INSTANCE, createApiKeyRequest).actionGet();
 
         final Map<String, Object> apiKeyDocument = getApiKeyDocument(createApiKeyResponse.getId());
 
@@ -153,6 +157,7 @@ public class AnonymousUserIntegTests extends SecurityIntegTestCase {
 
         final GrantApiKeyRequest grantApiKeyRequest = new GrantApiKeyRequest();
         grantApiKeyRequest.getApiKeyRequest().setName("granted-api-key-cannot-have-anonymous-user-token-with-run-as");
+        grantApiKeyRequest.setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL, NONE));
         grantApiKeyRequest.getGrant().setType("access_token");
         grantApiKeyRequest.getGrant().setAccessToken(new SecureString(oAuth2Token.accessToken().toCharArray()));
         grantApiKeyRequest.getGrant().setRunAsUsername("test_user");

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java

@@ -567,7 +567,7 @@ public class Security extends Plugin
         this.settings = settings;
         // TODO this is wrong, we should only use the environment that is provided to createComponents
         this.enabled = XPackSettings.SECURITY_ENABLED.get(settings);
-        this.systemIndices = new SecuritySystemIndices();
+        this.systemIndices = new SecuritySystemIndices(settings);
         this.nodeStartedListenable = new ListenableFuture<>();
         if (enabled) {
             runStartupChecks(settings);

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java

@@ -36,6 +36,7 @@ import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.action.update.UpdateResponse;
 import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
@@ -563,7 +564,7 @@ public class ApiKeyService {
         }
 
         logger.trace("Executing bulk request to update [{}] API keys", bulkRequestBuilder.numberOfActions());
-        bulkRequestBuilder.setRefreshPolicy(RefreshPolicy.WAIT_UNTIL);
+        bulkRequestBuilder.setRefreshPolicy(defaultCreateDocRefreshPolicy(settings));
         securityIndex.prepareIndexIfNeededThenExecute(
             ex -> listener.onFailure(traceLog("prepare security index before update", ex)),
             () -> executeAsyncWithOrigin(
@@ -1677,7 +1678,7 @@ public class ApiKeyService {
                     .request();
                 bulkRequestBuilder.add(request);
             }
-            bulkRequestBuilder.setRefreshPolicy(RefreshPolicy.WAIT_UNTIL);
+            bulkRequestBuilder.setRefreshPolicy(defaultCreateDocRefreshPolicy(settings));
             securityIndex.prepareIndexIfNeededThenExecute(
                 ex -> listener.onFailure(traceLog("prepare security index", ex)),
                 () -> executeAsyncWithOrigin(
@@ -2324,6 +2325,18 @@ public class ApiKeyService {
         }
     }
 
+    /**
+     * API Key documents are refreshed after creation, such that the API Key docs are visible in searches after the create-API-key
+     * endpoint returns.
+     * In stateful deployments, the automatic refresh interval is short (hard-coded to 1 sec), so the {@code RefreshPolicy#WAIT_UNTIL}
+     * is an acceptable tradeoff for the superior doc creation throughput compared to {@code RefreshPolicy#IMMEDIATE}.
+     * But in stateless the automatic refresh interval is too long (at least 10 sec), which translates to long create-API-key endpoint
+     * latency, so in this case we opt for {@code RefreshPolicy#IMMEDIATE} and acknowledge the lower maximum doc creation throughput.
+     */
+    public static RefreshPolicy defaultCreateDocRefreshPolicy(Settings settings) {
+        return DiscoveryNode.isStateless(settings) ? RefreshPolicy.IMMEDIATE : RefreshPolicy.WAIT_UNTIL;
+    }
+
     private static final class ApiKeyDocCache {
         private final Cache<String, ApiKeyService.CachedApiKeyDoc> docCache;
         private final Cache<String, BytesReference> roleDescriptorsBytesCache;

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/enrollment/InternalEnrollmentTokenGenerator.java

@@ -14,6 +14,7 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
 import org.elasticsearch.action.admin.cluster.node.info.NodesInfoAction;
 import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
+import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.client.internal.OriginSettingClient;
 import org.elasticsearch.common.UUIDs;
@@ -192,6 +193,7 @@ public class InternalEnrollmentTokenGenerator extends BaseEnrollmentTokenGenerat
             List.of(new RoleDescriptor("create_enrollment_token", new String[] { enrollTokenType.toString() }, null, null)),
             TimeValue.timeValueMinutes(ENROLL_API_KEY_EXPIRATION_MINUTES)
         );
+        apiKeyRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
         client.execute(CreateApiKeyAction.INSTANCE, apiKeyRequest, ActionListener.wrap(createApiKeyResponse -> {
             final String apiKey = createApiKeyResponse.getId() + ":" + createApiKeyResponse.getKey().toString();
             final EnrollmentToken enrollmentToken = new EnrollmentToken(apiKey, fingerprint, Version.CURRENT.toString(), tokenAddresses);

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestCreateApiKeyAction.java

@@ -15,8 +15,8 @@ import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
 import org.elasticsearch.rest.ServerlessScope;
 import org.elasticsearch.rest.action.RestToXContentListener;
-import org.elasticsearch.xpack.core.security.action.apikey.CreateApiKeyRequest;
 import org.elasticsearch.xpack.core.security.action.apikey.CreateApiKeyRequestBuilder;
+import org.elasticsearch.xpack.security.authc.ApiKeyService;
 
 import java.io.IOException;
 import java.util.List;
@@ -51,14 +51,16 @@ public final class RestCreateApiKeyAction extends ApiKeyBaseRestHandler {
 
     @Override
     protected RestChannelConsumer innerPrepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-        String refresh = request.param("refresh");
         CreateApiKeyRequestBuilder builder = new CreateApiKeyRequestBuilder(client).source(
             request.requiredContent(),
             request.getXContentType()
-        )
-            .setRefreshPolicy(
-                (refresh != null) ? WriteRequest.RefreshPolicy.parse(request.param("refresh")) : CreateApiKeyRequest.DEFAULT_REFRESH_POLICY
-            );
+        );
+        String refresh = request.param("refresh");
+        if (refresh != null) {
+            builder.setRefreshPolicy(WriteRequest.RefreshPolicy.parse(request.param("refresh")));
+        } else {
+            builder.setRefreshPolicy(ApiKeyService.defaultCreateDocRefreshPolicy(settings));
+        }
         return channel -> builder.execute(new RestToXContentListener<>(channel));
     }
 }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestCreateCrossClusterApiKeyAction.java

@@ -19,6 +19,7 @@ import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xpack.core.security.action.apikey.CreateCrossClusterApiKeyAction;
 import org.elasticsearch.xpack.core.security.action.apikey.CreateCrossClusterApiKeyRequest;
 import org.elasticsearch.xpack.core.security.action.apikey.CrossClusterApiKeyRoleDescriptorBuilder;
+import org.elasticsearch.xpack.security.authc.ApiKeyService;
 
 import java.io.IOException;
 import java.util.List;
@@ -75,6 +76,7 @@ public final class RestCreateCrossClusterApiKeyAction extends ApiKeyBaseRestHand
     @Override
     protected RestChannelConsumer innerPrepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         final CreateCrossClusterApiKeyRequest createCrossClusterApiKeyRequest = PARSER.parse(request.contentParser(), null);
+        createCrossClusterApiKeyRequest.setRefreshPolicy(ApiKeyService.defaultCreateDocRefreshPolicy(settings));
         return channel -> client.execute(
             CreateCrossClusterApiKeyAction.INSTANCE,
             createCrossClusterApiKeyRequest,

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestGrantApiKeyAction.java

@@ -27,6 +27,7 @@ import org.elasticsearch.xpack.core.security.action.apikey.CreateApiKeyRequestBu
 import org.elasticsearch.xpack.core.security.action.apikey.CreateApiKeyResponse;
 import org.elasticsearch.xpack.core.security.action.apikey.GrantApiKeyAction;
 import org.elasticsearch.xpack.core.security.action.apikey.GrantApiKeyRequest;
+import org.elasticsearch.xpack.security.authc.ApiKeyService;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -94,6 +95,8 @@ public final class RestGrantApiKeyAction extends ApiKeyBaseRestHandler implement
             final GrantApiKeyRequest grantRequest = PARSER.parse(parser, null);
             if (refresh != null) {
                 grantRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.parse(refresh));
+            } else {
+                grantRequest.setRefreshPolicy(ApiKeyService.defaultCreateDocRefreshPolicy(settings));
             }
             return channel -> client.execute(
                 GrantApiKeyAction.INSTANCE,

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java

@@ -12,8 +12,10 @@ import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.indices.ExecutorNames;
 import org.elasticsearch.indices.SystemIndexDescriptor;
 import org.elasticsearch.transport.TcpTransport;
@@ -58,10 +60,10 @@ public class SecuritySystemIndices {
     private SecurityIndexManager tokenIndexManager;
     private SecurityIndexManager profileIndexManager;
 
-    public SecuritySystemIndices() {
+    public SecuritySystemIndices(Settings settings) {
         this.mainDescriptor = getSecurityMainIndexDescriptor();
         this.tokenDescriptor = getSecurityTokenIndexDescriptor();
-        this.profileDescriptor = getSecurityProfileIndexDescriptor();
+        this.profileDescriptor = getSecurityProfileIndexDescriptor(settings);
         this.initialized = new AtomicBoolean(false);
         this.mainIndexManager = null;
         this.tokenIndexManager = null;
@@ -778,13 +780,13 @@ public class SecuritySystemIndices {
         }
     }
 
-    private SystemIndexDescriptor getSecurityProfileIndexDescriptor() {
+    private SystemIndexDescriptor getSecurityProfileIndexDescriptor(Settings settings) {
         return SystemIndexDescriptor.builder()
             .setIndexPattern(".security-profile-[0-9]+*")
             .setPrimaryIndex(INTERNAL_SECURITY_PROFILE_INDEX_8)
             .setDescription("Contains user profile documents")
             .setMappings(getProfileIndexMappings())
-            .setSettings(getProfileIndexSettings())
+            .setSettings(getProfileIndexSettings(settings))
             .setAliasName(SECURITY_PROFILE_ALIAS)
             .setIndexFormat(INTERNAL_PROFILE_INDEX_FORMAT)
             .setVersionMetaKey(SECURITY_VERSION_STRING)
@@ -798,7 +800,7 @@ public class SecuritySystemIndices {
                         .setPrimaryIndex(INTERNAL_SECURITY_PROFILE_INDEX_8)
                         .setDescription("Contains user profile documents")
                         .setMappings(getProfileIndexMappings())
-                        .setSettings(getProfileIndexSettings())
+                        .setSettings(getProfileIndexSettings(settings))
                         .setAliasName(SECURITY_PROFILE_ALIAS)
                         .setIndexFormat(INTERNAL_PROFILE_INDEX_FORMAT)
                         .setVersionMetaKey(SECURITY_VERSION_STRING)
@@ -810,8 +812,8 @@ public class SecuritySystemIndices {
             .build();
     }
 
-    private static Settings getProfileIndexSettings() {
-        return Settings.builder()
+    private static Settings getProfileIndexSettings(Settings settings) {
+        final Settings.Builder settingsBuilder = Settings.builder()
             .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
             .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
             .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
@@ -821,8 +823,15 @@ public class SecuritySystemIndices {
             .put("analysis.filter.email.preserve_original", true)
             .putList("analysis.filter.email.patterns", List.of("([^@]+)", "(\\p{L}+)", "(\\d+)", "@(.+)"))
             .put("analysis.analyzer.email.tokenizer", "uax_url_email")
-            .putList("analysis.analyzer.email.filter", List.of("email", "lowercase", "unique"))
-            .build();
+            .putList("analysis.analyzer.email.filter", List.of("email", "lowercase", "unique"));
+        if (DiscoveryNode.isStateless(settings)) {
+            // The profiles functionality is intrinsically related to Kibana. Only Kibana uses this index (via dedicated APIs).
+            // Since the regular ".kibana" index is marked "fast_refresh", we opt to mark the user profiles index as "fast_refresh" too.
+            // This way the profiles index has the same availability and latency characteristics as the regular ".kibana" index, so APIs
+            // touching either of the two indices are more predictable.
+            settingsBuilder.put(IndexSettings.INDEX_FAST_REFRESH_SETTING.getKey(), true);
+        }
+        return settingsBuilder.build();
     }
 
     private XContentBuilder getProfileIndexMappings() {

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java

@@ -306,7 +306,7 @@ public class NativeRolesStoreTests extends ESTestCase {
         final XPackLicenseState licenseState = mock(XPackLicenseState.class);
         final AtomicBoolean methodCalled = new AtomicBoolean(false);
 
-        final SecuritySystemIndices systemIndices = new SecuritySystemIndices();
+        final SecuritySystemIndices systemIndices = new SecuritySystemIndices(clusterService.getSettings());
         systemIndices.init(client, clusterService);
         final SecurityIndexManager securityIndex = systemIndices.getMainIndexManager();
 
@@ -394,7 +394,7 @@ public class NativeRolesStoreTests extends ESTestCase {
         final XPackLicenseState licenseState = mock(XPackLicenseState.class);
         final AtomicBoolean methodCalled = new AtomicBoolean(false);
 
-        final SecuritySystemIndices systemIndices = new SecuritySystemIndices();
+        final SecuritySystemIndices systemIndices = new SecuritySystemIndices(clusterService.getSettings());
         systemIndices.init(client, clusterService);
         final SecurityIndexManager securityIndex = systemIndices.getMainIndexManager();
 
@@ -441,6 +441,7 @@ public class NativeRolesStoreTests extends ESTestCase {
     private ClusterService mockClusterServiceWithMinNodeVersion(TransportVersion transportVersion) {
         final ClusterService clusterService = mock(ClusterService.class, Mockito.RETURNS_DEEP_STUBS);
         when(clusterService.state().getMinTransportVersion()).thenReturn(transportVersion);
+        when(clusterService.getSettings()).thenReturn(Settings.EMPTY);
         return clusterService;
     }
 

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessAuthenticationServiceIntegTests.java

@@ -35,6 +35,9 @@ import java.util.Base64;
 import java.util.concurrent.ExecutionException;
 import java.util.function.Consumer;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.NONE;
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.WAIT_UNTIL;
 import static org.elasticsearch.xpack.core.security.authc.CrossClusterAccessSubjectInfo.CROSS_CLUSTER_ACCESS_SUBJECT_INFO_HEADER_KEY;
 import static org.elasticsearch.xpack.security.authc.CrossClusterAccessHeaders.CROSS_CLUSTER_ACCESS_CREDENTIALS_HEADER_KEY;
 import static org.hamcrest.Matchers.containsString;
@@ -142,6 +145,7 @@ public class CrossClusterAccessAuthenticationServiceIntegTests extends SecurityI
     private String getEncodedCrossClusterAccessApiKey() throws IOException {
         final CreateCrossClusterApiKeyRequest request = CreateCrossClusterApiKeyRequest.withNameAndAccess("cross_cluster_access_key", """
             {"search": [{"names": ["*"]}]}""");
+        request.setRefreshPolicy(randomFrom(NONE, IMMEDIATE, WAIT_UNTIL));
         final CreateApiKeyResponse response = client().execute(CreateCrossClusterApiKeyAction.INSTANCE, request).actionGet();
         return ApiKeyService.withApiKeyPrefix(
             Base64.getEncoder().encodeToString((response.getId() + ":" + response.getKey()).getBytes(StandardCharsets.UTF_8))

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java

@@ -103,7 +103,8 @@ public class SecurityIndexManagerTests extends ESTestCase {
         };
 
         final ClusterService clusterService = mock(ClusterService.class);
-        final SystemIndexDescriptor descriptor = new SecuritySystemIndices().getSystemIndexDescriptors()
+        when(clusterService.getSettings()).thenReturn(Settings.EMPTY);
+        final SystemIndexDescriptor descriptor = new SecuritySystemIndices(clusterService.getSettings()).getSystemIndexDescriptors()
             .stream()
             .filter(d -> d.getAliasName().equals(SecuritySystemIndices.SECURITY_MAIN_ALIAS))
             .findFirst()