Fix auditing of API Key authn without the owner realm name (#59470)

The `Authentication` object that gets built following an API Key authentication
contains the realm name of the owner user that created the key (which is audited),
but the specific field used for storing it changed in #51305 .

This PR makes it so that auditing tolerates an "unfound" realm name, so it doesn't
throw an NPE, because the owner realm name is not found in the expected field.

Closes #59425

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java

@@ -239,7 +239,8 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
         if (events.contains(AUTHENTICATION_SUCCESS) && eventFilterPolicyRegistry.ignorePredicate()
                 .test(new AuditEventMetaInfo(
                         Optional.of(authentication.getUser()),
-                        Optional.of(ApiKeyService.getCreatorRealmName(authentication)),
+                        // can be null for API keys created before version 7.7
+                        Optional.ofNullable(ApiKeyService.getCreatorRealmName(authentication)),
                         Optional.empty(),
                         Optional.empty())) == false) {
             // this is redundant information maintained for bwc purposes
@@ -267,7 +268,8 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
             if (eventFilterPolicyRegistry.ignorePredicate()
                     .test(new AuditEventMetaInfo(
                             Optional.of(authentication.getUser()),
-                            Optional.of(ApiKeyService.getCreatorRealmName(authentication)),
+                            // can be null for API keys created before version 7.7
+                            Optional.ofNullable(ApiKeyService.getCreatorRealmName(authentication)),
                             Optional.empty(),
                             indices)) == false) {
                 final StringMapMessage logEntry = new LogEntryBuilder()
@@ -461,7 +463,9 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
         if ((isSystem && events.contains(SYSTEM_ACCESS_GRANTED)) || ((isSystem == false) && events.contains(ACCESS_GRANTED))) {
             final Optional<String[]> indices = indices(msg);
             if (eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.of(user),
-                    Optional.of(ApiKeyService.getCreatorRealmName(authentication)), Optional.of(authorizationInfo), indices)) == false) {
+                    // can be null for API keys created before version 7.7
+                    Optional.ofNullable(ApiKeyService.getCreatorRealmName(authentication)),
+                    Optional.of(authorizationInfo), indices)) == false) {
                 final StringMapMessage logEntry = new LogEntryBuilder()
                         .with(EVENT_TYPE_FIELD_NAME, TRANSPORT_ORIGIN_FIELD_VALUE)
                         .with(EVENT_ACTION_FIELD_NAME, "access_granted")
@@ -491,7 +495,9 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
         }
         if (events.contains(eventType)) {
             if (eventFilterPolicyRegistry.ignorePredicate()
-                    .test(new AuditEventMetaInfo(Optional.of(user), Optional.of(ApiKeyService.getCreatorRealmName(authentication)),
+                    .test(new AuditEventMetaInfo(Optional.of(user),
+                            // can be null for API keys created before version 7.7
+                            Optional.ofNullable(ApiKeyService.getCreatorRealmName(authentication)),
                             Optional.of(authorizationInfo), Optional.ofNullable(indices))) == false) {
                 final LogEntryBuilder logEntryBuilder = new LogEntryBuilder()
                         .with(EVENT_TYPE_FIELD_NAME, TRANSPORT_ORIGIN_FIELD_VALUE)
@@ -525,7 +531,9 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
         if (events.contains(ACCESS_DENIED)) {
             final Optional<String[]> indices = indices(transportRequest);
             if (eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.of(authentication.getUser()),
-                    Optional.of(ApiKeyService.getCreatorRealmName(authentication)), Optional.of(authorizationInfo), indices)) == false) {
+                    // can be null for API keys created before version 7.7
+                    Optional.ofNullable(ApiKeyService.getCreatorRealmName(authentication)),
+                    Optional.of(authorizationInfo), indices)) == false) {
                 final StringMapMessage logEntry = new LogEntryBuilder()
                         .with(EVENT_TYPE_FIELD_NAME, TRANSPORT_ORIGIN_FIELD_VALUE)
                         .with(EVENT_ACTION_FIELD_NAME, "access_denied")
@@ -589,7 +597,8 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
             final Optional<String[]> indices = indices(transportRequest);
             if (eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(
                             Optional.of(authentication.getUser()),
-                            Optional.of(ApiKeyService.getCreatorRealmName(authentication)),
+                            // can be null for API keys created before version 7.7
+                            Optional.ofNullable(ApiKeyService.getCreatorRealmName(authentication)),
                             Optional.empty(),
                             indices)) == false) {
                 final StringMapMessage logEntry = new LogEntryBuilder()
@@ -651,7 +660,9 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
         if (events.contains(RUN_AS_GRANTED)) {
             final Optional<String[]> indices = indices(transportRequest);
             if (eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.of(authentication.getUser()),
-                    Optional.of(ApiKeyService.getCreatorRealmName(authentication)), Optional.of(authorizationInfo), indices)) == false) {
+                    // can be null for API keys created before version 7.7
+                    Optional.ofNullable(ApiKeyService.getCreatorRealmName(authentication)),
+                    Optional.of(authorizationInfo), indices)) == false) {
                 final StringMapMessage logEntry = new LogEntryBuilder()
                         .with(EVENT_TYPE_FIELD_NAME, TRANSPORT_ORIGIN_FIELD_VALUE)
                         .with(EVENT_ACTION_FIELD_NAME, "run_as_granted")
@@ -676,7 +687,9 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
         if (events.contains(RUN_AS_DENIED)) {
             final Optional<String[]> indices = indices(transportRequest);
             if (eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.of(authentication.getUser()),
-                    Optional.of(ApiKeyService.getCreatorRealmName(authentication)), Optional.of(authorizationInfo), indices)) == false) {
+                    // can be null for API keys created before version 7.7
+                    Optional.ofNullable(ApiKeyService.getCreatorRealmName(authentication)),
+                    Optional.of(authorizationInfo), indices)) == false) {
                 final StringMapMessage logEntry = new LogEntryBuilder()
                         .with(EVENT_TYPE_FIELD_NAME, TRANSPORT_ORIGIN_FIELD_VALUE)
                         .with(EVENT_ACTION_FIELD_NAME, "run_as_denied")
@@ -699,7 +712,8 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
     public void runAsDenied(String requestId, Authentication authentication, RestRequest request, AuthorizationInfo authorizationInfo) {
         if (events.contains(RUN_AS_DENIED) && eventFilterPolicyRegistry.ignorePredicate().test(
                 new AuditEventMetaInfo(Optional.of(authentication.getUser()),
-                        Optional.of(ApiKeyService.getCreatorRealmName(authentication)),
+                        // can be null for API keys created before version 7.7
+                        Optional.ofNullable(ApiKeyService.getCreatorRealmName(authentication)),
                         Optional.of(authorizationInfo), Optional.empty())) == false) {
             final StringMapMessage logEntry = new LogEntryBuilder()
                     .with(EVENT_TYPE_FIELD_NAME, REST_ORIGIN_FIELD_VALUE)
@@ -819,9 +833,12 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
             logEntry.with(AUTHENTICATION_TYPE_FIELD_NAME, authentication.getAuthenticationType().toString());
             if (Authentication.AuthenticationType.API_KEY == authentication.getAuthenticationType()) {
                 logEntry.with(API_KEY_ID_FIELD_NAME, (String) authentication.getMetadata().get(ApiKeyService.API_KEY_ID_KEY))
-                        .with(API_KEY_NAME_FIELD_NAME, (String) authentication.getMetadata().get(ApiKeyService.API_KEY_NAME_KEY))
-                        .with(PRINCIPAL_REALM_FIELD_NAME,
-                                (String) authentication.getMetadata().get(ApiKeyService.API_KEY_CREATOR_REALM_NAME));
+                        .with(API_KEY_NAME_FIELD_NAME, (String) authentication.getMetadata().get(ApiKeyService.API_KEY_NAME_KEY));
+                String creatorRealmName = (String) authentication.getMetadata().get(ApiKeyService.API_KEY_CREATOR_REALM_NAME);
+                if (creatorRealmName != null) {
+                    // can be null for API keys created before version 7.7
+                    logEntry.with(PRINCIPAL_REALM_FIELD_NAME, creatorRealmName);
+                }
             } else {
                 if (authentication.getUser().isRunAs()) {
                     logEntry.with(PRINCIPAL_REALM_FIELD_NAME, authentication.getLookedUpBy().getName())

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailFilterTests.java

@@ -865,19 +865,35 @@ public class LoggingAuditTrailFilterTests extends ESTestCase {
         threadContext.stashContext();
 
         // accessGranted
-        auditTrail.accessGranted(randomAlphaOfLength(8),
-                randomBoolean() ? createAuthentication(user, filteredRealm) :
-                        createApiKeyAuthentication(apiKeyService, createAuthentication(user, filteredRealm)),
-                "_action", request, authzInfo(new String[]{"role1"}));
-        assertThat("AccessGranted message: filtered realm is not filtered out", logOutput.size(), is(0));
+        Authentication authentication = randomBoolean() ? createAuthentication(user, filteredRealm) :
+                createApiKeyAuthentication(apiKeyService, createAuthentication(user, filteredRealm));
+        auditTrail.accessGranted(randomAlphaOfLength(8), authentication, "_action", request, authzInfo(new String[]{"role1"}));
+        if (authentication.getAuthenticationType() == Authentication.AuthenticationType.API_KEY &&
+                false == authentication.getMetadata().containsKey(ApiKeyService.API_KEY_CREATOR_REALM_NAME)) {
+            if (filterMissingRealm) {
+                assertThat("AccessGranted message: not filtered out by the missing realm filter", logOutput.size(), is(0));
+            } else {
+                assertThat("AccessGranted message: filtered out by the realm filters", logOutput.size(), is(1));
+            }
+        } else {
+            assertThat("AccessGranted message: filtered realm is not filtered out", logOutput.size(), is(0));
+        }
         logOutput.clear();
         threadContext.stashContext();
 
-        auditTrail.accessGranted(randomAlphaOfLength(8),
-                randomBoolean() ? createAuthentication(user, unfilteredRealm) :
-                        createApiKeyAuthentication(apiKeyService, createAuthentication(user, unfilteredRealm)),
-                "_action", request, authzInfo(new String[]{"role1"}));
-        assertThat("AccessGranted message: unfiltered realm is filtered out", logOutput.size(), is(1));
+        authentication = randomBoolean() ? createAuthentication(user, unfilteredRealm) :
+                createApiKeyAuthentication(apiKeyService, createAuthentication(user, unfilteredRealm));
+        auditTrail.accessGranted(randomAlphaOfLength(8), authentication, "_action", request, authzInfo(new String[]{"role1"}));
+        if (authentication.getAuthenticationType() == Authentication.AuthenticationType.API_KEY &&
+                false == authentication.getMetadata().containsKey(ApiKeyService.API_KEY_CREATOR_REALM_NAME)) {
+            if (filterMissingRealm) {
+                assertThat("AccessGranted message: not filtered out by the missing realm filter", logOutput.size(), is(0));
+            } else {
+                assertThat("AccessGranted message: filtered out by the realm filters", logOutput.size(), is(1));
+            }
+        } else {
+            assertThat("AccessGranted message: unfiltered realm is filtered out", logOutput.size(), is(1));
+        }
         logOutput.clear();
         threadContext.stashContext();
 
@@ -893,32 +909,68 @@ public class LoggingAuditTrailFilterTests extends ESTestCase {
         logOutput.clear();
         threadContext.stashContext();
 
-        auditTrail.accessGranted(randomAlphaOfLength(8), randomBoolean() ? createAuthentication(user, filteredRealm) :
-                        createApiKeyAuthentication(apiKeyService, createAuthentication(user, filteredRealm)),
-                "internal:_action", request, authzInfo(new String[]{"role1"}));
-        assertThat("AccessGranted internal message: filtered realm is not filtered out", logOutput.size(), is(0));
+        authentication = randomBoolean() ? createAuthentication(user, filteredRealm) :
+                createApiKeyAuthentication(apiKeyService, createAuthentication(user, filteredRealm));
+        auditTrail.accessGranted(randomAlphaOfLength(8), authentication, "internal:_action", request, authzInfo(new String[]{"role1"}));
+        if (authentication.getAuthenticationType() == Authentication.AuthenticationType.API_KEY &&
+                false == authentication.getMetadata().containsKey(ApiKeyService.API_KEY_CREATOR_REALM_NAME)) {
+            if (filterMissingRealm) {
+                assertThat("AccessGranted internal message: not filtered out by the missing realm filter", logOutput.size(), is(0));
+            } else {
+                assertThat("AccessGranted internal message: filtered out by the realm filters", logOutput.size(), is(1));
+            }
+        } else {
+            assertThat("AccessGranted internal message: filtered realm is not filtered out", logOutput.size(), is(0));
+        }
         logOutput.clear();
         threadContext.stashContext();
 
-        auditTrail.accessGranted(randomAlphaOfLength(8), randomBoolean() ? createAuthentication(user, unfilteredRealm) :
-                createApiKeyAuthentication(apiKeyService, createAuthentication(user, unfilteredRealm)),
-                "internal:_action", request, authzInfo(new String[] { "role1" }));
-        assertThat("AccessGranted internal message: unfiltered realm is filtered out", logOutput.size(), is(1));
+        authentication = randomBoolean() ? createAuthentication(user, unfilteredRealm) :
+                createApiKeyAuthentication(apiKeyService, createAuthentication(user, unfilteredRealm));
+        auditTrail.accessGranted(randomAlphaOfLength(8), authentication, "internal:_action", request, authzInfo(new String[] { "role1" }));
+        if (authentication.getAuthenticationType() == Authentication.AuthenticationType.API_KEY &&
+                false == authentication.getMetadata().containsKey(ApiKeyService.API_KEY_CREATOR_REALM_NAME)) {
+            if (filterMissingRealm) {
+                assertThat("AccessGranted internal message: not filtered out by the missing realm filter", logOutput.size(), is(0));
+            } else {
+                assertThat("AccessGranted internal message: filtered out by the realm filters", logOutput.size(), is(1));
+            }
+        } else {
+            assertThat("AccessGranted internal message: unfiltered realm is filtered out", logOutput.size(), is(1));
+        }
         logOutput.clear();
         threadContext.stashContext();
 
         // accessDenied
-        auditTrail.accessDenied(randomAlphaOfLength(8), randomBoolean() ? createAuthentication(user, filteredRealm) :
-                        createApiKeyAuthentication(apiKeyService, createAuthentication(user, filteredRealm)), "_action", request,
-                authzInfo(new String[]{"role1"}));
-        assertThat("AccessDenied message: filtered realm is not filtered out", logOutput.size(), is(0));
+        authentication = randomBoolean() ? createAuthentication(user, filteredRealm) :
+                createApiKeyAuthentication(apiKeyService, createAuthentication(user, filteredRealm));
+        auditTrail.accessDenied(randomAlphaOfLength(8), authentication, "_action", request, authzInfo(new String[]{"role1"}));
+        if (authentication.getAuthenticationType() == Authentication.AuthenticationType.API_KEY &&
+                false == authentication.getMetadata().containsKey(ApiKeyService.API_KEY_CREATOR_REALM_NAME)) {
+            if (filterMissingRealm) {
+                assertThat("AccessDenied message: not filtered out by the missing realm filter", logOutput.size(), is(0));
+            } else {
+                assertThat("AccessDenied message: filtered out by the realm filters", logOutput.size(), is(1));
+            }
+        } else {
+            assertThat("AccessDenied message: filtered realm is not filtered out", logOutput.size(), is(0));
+        }
         logOutput.clear();
         threadContext.stashContext();
 
-        auditTrail.accessDenied(randomAlphaOfLength(8), randomBoolean() ? createAuthentication(user, unfilteredRealm) :
-                        createApiKeyAuthentication(apiKeyService, createAuthentication(user, unfilteredRealm)), "_action", request,
-                authzInfo(new String[]{"role1"}));
-        assertThat("AccessDenied message: unfiltered realm is filtered out", logOutput.size(), is(1));
+        authentication = randomBoolean() ? createAuthentication(user, unfilteredRealm) :
+                createApiKeyAuthentication(apiKeyService, createAuthentication(user, unfilteredRealm));
+        auditTrail.accessDenied(randomAlphaOfLength(8), authentication, "_action", request, authzInfo(new String[]{"role1"}));
+        if (authentication.getAuthenticationType() == Authentication.AuthenticationType.API_KEY &&
+                false == authentication.getMetadata().containsKey(ApiKeyService.API_KEY_CREATOR_REALM_NAME)) {
+            if (filterMissingRealm) {
+                assertThat("AccessDenied message: not filtered out by the missing realm filter", logOutput.size(), is(0));
+            } else {
+                assertThat("AccessDenied message: filtered out by the realm filters", logOutput.size(), is(1));
+            }
+        } else {
+            assertThat("AccessDenied message: unfiltered realm is filtered out", logOutput.size(), is(1));
+        }
         logOutput.clear();
         threadContext.stashContext();
 
@@ -934,17 +986,36 @@ public class LoggingAuditTrailFilterTests extends ESTestCase {
         logOutput.clear();
         threadContext.stashContext();
 
-        auditTrail.accessDenied(randomAlphaOfLength(8), randomBoolean() ? createAuthentication(user, filteredRealm) :
-                        createApiKeyAuthentication(apiKeyService, createAuthentication(user, filteredRealm)), "internal:_action",
-                request, authzInfo(new String[]{"role1"}));
-        assertThat("AccessGranted internal message: filtered realm is not filtered out", logOutput.size(), is(0));
+        authentication = randomBoolean() ? createAuthentication(user, filteredRealm) :
+                createApiKeyAuthentication(apiKeyService, createAuthentication(user, filteredRealm));
+        auditTrail.accessDenied(randomAlphaOfLength(8), authentication, "internal:_action", request, authzInfo(new String[]{"role1"}));
+        if (authentication.getAuthenticationType() == Authentication.AuthenticationType.API_KEY &&
+                false == authentication.getMetadata().containsKey(ApiKeyService.API_KEY_CREATOR_REALM_NAME)) {
+            if (filterMissingRealm) {
+                assertThat("AccessDenied internal message: not filtered out by the missing realm filter", logOutput.size(), is(0));
+            } else {
+                assertThat("AccessDenied internal message: filtered out by the realm filters", logOutput.size(), is(1));
+            }
+        } else {
+            assertThat("AccessDenied internal message: filtered realm is filtered out", logOutput.size(), is(0));
+        }
         logOutput.clear();
         threadContext.stashContext();
 
-        auditTrail.accessDenied(randomAlphaOfLength(8), randomBoolean() ? createAuthentication(user, unfilteredRealm) :
-                        createApiKeyAuthentication(apiKeyService, createAuthentication(user, unfilteredRealm)), "internal:_action",
+        authentication = randomBoolean() ? createAuthentication(user, unfilteredRealm) :
+                createApiKeyAuthentication(apiKeyService, createAuthentication(user, unfilteredRealm));
+        auditTrail.accessDenied(randomAlphaOfLength(8), authentication, "internal:_action",
                 request, authzInfo(new String[]{"role1"}));
-        assertThat("AccessGranted internal message: unfiltered realm is filtered out", logOutput.size(), is(1));
+        if (authentication.getAuthenticationType() == Authentication.AuthenticationType.API_KEY &&
+                false == authentication.getMetadata().containsKey(ApiKeyService.API_KEY_CREATOR_REALM_NAME)) {
+            if (filterMissingRealm) {
+                assertThat("AccessDenied internal message: not filtered out by the missing realm filter", logOutput.size(), is(0));
+            } else {
+                assertThat("AccessDenied internal message: filtered out by the realm filters", logOutput.size(), is(1));
+            }
+        } else {
+            assertThat("AccessDenied internal message: unfiltered realm is filtered out", logOutput.size(), is(1));
+        }
         logOutput.clear();
         threadContext.stashContext();
 
@@ -967,15 +1038,35 @@ public class LoggingAuditTrailFilterTests extends ESTestCase {
         logOutput.clear();
         threadContext.stashContext();
 
-        auditTrail.tamperedRequest(randomAlphaOfLength(8), randomBoolean() ? createAuthentication(user, filteredRealm) :
-                createApiKeyAuthentication(apiKeyService, createAuthentication(user, filteredRealm)), "_action", request);
-        assertThat("Tampered message: filtered realm is not filtered out", logOutput.size(), is(0));
+        authentication = randomBoolean() ? createAuthentication(user, filteredRealm) :
+                createApiKeyAuthentication(apiKeyService, createAuthentication(user, filteredRealm));
+        auditTrail.tamperedRequest(randomAlphaOfLength(8), authentication, "_action", request);
+        if (authentication.getAuthenticationType() == Authentication.AuthenticationType.API_KEY &&
+                false == authentication.getMetadata().containsKey(ApiKeyService.API_KEY_CREATOR_REALM_NAME)) {
+            if (filterMissingRealm) {
+                assertThat("Tampered message: not filtered out by the missing realm filter", logOutput.size(), is(0));
+            } else {
+                assertThat("Tampered message: filtered out by the realm filters", logOutput.size(), is(1));
+            }
+        } else {
+            assertThat("Tampered message: filtered realm is not filtered out", logOutput.size(), is(0));
+        }
         logOutput.clear();
         threadContext.stashContext();
 
-        auditTrail.tamperedRequest(randomAlphaOfLength(8), randomBoolean() ? createAuthentication(user, unfilteredRealm) :
-                createApiKeyAuthentication(apiKeyService, createAuthentication(user, unfilteredRealm)), "_action", request);
-        assertThat("Tampered message: unfiltered realm is filtered out", logOutput.size(), is(1));
+        authentication = randomBoolean() ? createAuthentication(user, unfilteredRealm) :
+                createApiKeyAuthentication(apiKeyService, createAuthentication(user, unfilteredRealm));
+        auditTrail.tamperedRequest(randomAlphaOfLength(8), authentication, "_action", request);
+        if (authentication.getAuthenticationType() == Authentication.AuthenticationType.API_KEY &&
+                false == authentication.getMetadata().containsKey(ApiKeyService.API_KEY_CREATOR_REALM_NAME)) {
+            if (filterMissingRealm) {
+                assertThat("Tampered message: not filtered out by the missing realm filter", logOutput.size(), is(0));
+            } else {
+                assertThat("Tampered message: filtered out by the realm filters", logOutput.size(), is(1));
+            }
+        } else {
+            assertThat("Tampered message: unfiltered realm is filtered out", logOutput.size(), is(1));
+        }
         logOutput.clear();
         threadContext.stashContext();
 

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java

@@ -1463,9 +1463,12 @@ public class LoggingAuditTrailTests extends ESTestCase {
             checkedFields.put(LoggingAuditTrail.API_KEY_ID_FIELD_NAME,
                     (String) authentication.getMetadata().get(ApiKeyService.API_KEY_ID_KEY))
                     .put(LoggingAuditTrail.API_KEY_NAME_FIELD_NAME,
-                            (String) authentication.getMetadata().get(ApiKeyService.API_KEY_NAME_KEY))
-                    .put(LoggingAuditTrail.PRINCIPAL_REALM_FIELD_NAME,
-                            (String) authentication.getMetadata().get(ApiKeyService.API_KEY_CREATOR_REALM_NAME));
+                            (String) authentication.getMetadata().get(ApiKeyService.API_KEY_NAME_KEY));
+            String creatorRealmName = (String) authentication.getMetadata().get(ApiKeyService.API_KEY_CREATOR_REALM_NAME);
+            if (creatorRealmName != null) {
+                checkedFields.put(LoggingAuditTrail.PRINCIPAL_REALM_FIELD_NAME, creatorRealmName);
+            }
+
         } else {
             if (authentication.getUser().isRunAs()) {
                 checkedFields.put(LoggingAuditTrail.PRINCIPAL_REALM_FIELD_NAME, authentication.getLookedUpBy().getName())

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java

@@ -845,6 +845,7 @@ public class ApiKeyServiceTests extends ESTestCase {
     }
 
     public static class Utils {
+
         private static final AuthenticationContextSerializer authenticationContextSerializer = new AuthenticationContextSerializer();
 
         public static Authentication createApiKeyAuthentication(ApiKeyService apiKeyService,
@@ -862,24 +863,27 @@ public class ApiKeyServiceTests extends ESTestCase {
                             new SecureString("pass".toCharArray())),
                     Clock.systemUTC(), authenticationResultFuture);
 
-            final TestThreadPool threadPool = new TestThreadPool("utils");
-            try {
-                final ThreadContext threadContext = threadPool.getThreadContext();
-                final SecurityContext securityContext = new SecurityContext(Settings.EMPTY, threadContext);
-                authenticationContextSerializer.writeToContext(
-                    apiKeyService.createApiKeyAuthentication(authenticationResultFuture.get(), "node01"), threadContext);
-                final CompletableFuture<Authentication> authFuture = new CompletableFuture<>();
-                securityContext.executeAfterRewritingAuthentication((c) -> {
-                    try {
-                        authFuture.complete(authenticationContextSerializer.readFromContext(threadContext));
-                    } catch (IOException e) {
-                        throw new RuntimeException(e);
-                    }
-                }, version);
-                return authFuture.get();
-            } finally {
-                terminate(threadPool);
+            AuthenticationResult authenticationResult = authenticationResultFuture.get();
+            if (randomBoolean()) {
+                // maybe remove realm name to simulate old API Key authentication
+                Map<String, Object> authenticationResultMetadata = new HashMap<>(authenticationResult.getMetadata());
+                authenticationResultMetadata.remove(ApiKeyService.API_KEY_CREATOR_REALM_NAME);
+                authenticationResult = AuthenticationResult.success(authenticationResult.getUser(), authenticationResultMetadata);
             }
+
+            final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
+            final SecurityContext securityContext = new SecurityContext(Settings.EMPTY, threadContext);
+            authenticationContextSerializer.writeToContext(
+                    apiKeyService.createApiKeyAuthentication(authenticationResult, "node01"), threadContext);
+            final CompletableFuture<Authentication> authFuture = new CompletableFuture<>();
+            securityContext.executeAfterRewritingAuthentication((c) -> {
+                try {
+                    authFuture.complete(authenticationContextSerializer.readFromContext(threadContext));
+                } catch (IOException e) {
+                    throw new RuntimeException(e);
+                }
+            }, version);
+            return authFuture.get();
         }
 
         public static Authentication createApiKeyAuthentication(ApiKeyService apiKeyService,

x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/120_api_key_auth.yml

@@ -3,8 +3,6 @@
 
   - skip:
       features: headers
-      version: " - 7.99.99"
-      reason: "7.9.0 backport pending"
 
   - do:
       security.create_api_key: