Expose API Key metadata to SetSecurityUser ingest processor (#72137)

This PR ensures SetSecurityUserProcessor adds the API key metadata
inside the existing api_key object if the metadata is not null or empty.

docs/reference/ingest/processors/set-security-user.asciidoc

@@ -8,7 +8,8 @@ Sets user-related details (such as `username`,  `roles`, `email`, `full_name`,
 `metadata`, `api_key`, `realm` and `authentication_type`) from the current
 authenticated user to the current document by pre-processing the ingest.
 The `api_key` property exists only if the user authenticates with an
-API key. It is an object containing the `id` and `name` fields of the API key.
+API key. It is an object containing the `id`, `name` and `metadata`
+(if it exists and is non-empty) fields of the API key.
 The `realm` property is also an object with two fields, `name` and `type`.
 When using API key authentication, the `realm` property refers to the realm
 from which the API key is created.

x-pack/docs/en/security/authorization/set-security-user.asciidoc

@@ -3,7 +3,7 @@
 
 // If an index is shared by many small users it makes sense to put all these users
 // into the same index. Having a dedicated index or shard per user is wasteful.
-// TBD: It's unclear why we're putting users in an index here. 
+// TBD: It's unclear why we're putting users in an index here.
 
 To guarantee that a user reads only their own documents, it makes sense to set up
 document level security. In this scenario, each document must have the username
@@ -20,8 +20,9 @@ The <<ingest-node-set-security-user-processor,set security user processor>> atta
 `username`,  `roles`, `email`, `full_name` and `metadata` ) from the current
 authenticated user to the current document by pre-processing the ingest. When
 you index data with an ingest pipeline, user details are automatically attached
-to the document.
+to the document. If the authenticating credential is an API key, the API key
+`id`, `name` and `metadata` (if it exists and is non-empty) are also attached to
+the document.
 
 For more information see <<ingest>> and
 <<ingest-node-set-security-user-processor>>
-

x-pack/plugin/security/qa/security-disabled/build.gradle

@@ -1,5 +1,5 @@
 /*
- * This QA project tests the security plugin when security is explicitlt disabled.
+ * This QA project tests the security plugin when security is explicitly disabled.
  * It is intended to cover security functionality which is supposed to
  * function in a specific way even if security is disabled on the cluster
  * For example: If a cluster has a pipeline with the set_security_user processor

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java

@@ -1119,6 +1119,27 @@ public class ApiKeyService {
         }
     }
 
+    /**
+     * If the authentication has type of api_key, returns the metadata associated to the
+     * API key.
+     * @param authentication {@link Authentication}
+     * @return A map for the metadata or an empty map if no metadata is found.
+     */
+    public static Map<String, Object> getApiKeyMetadata(Authentication authentication) {
+        if (AuthenticationType.API_KEY != authentication.getAuthenticationType()) {
+            throw new IllegalArgumentException("authentication type must be [api_key], got ["
+                + authentication.getAuthenticationType().name().toLowerCase(Locale.ROOT) + "]");
+        }
+        final Object apiKeyMetadata = authentication.getMetadata().get(ApiKeyService.API_KEY_METADATA_KEY);
+        if (apiKeyMetadata != null) {
+            final Tuple<XContentType, Map<String, Object>> tuple =
+                XContentHelper.convertToMap((BytesReference) apiKeyMetadata, false, XContentType.JSON);
+            return tuple.v2();
+        } else {
+            return Map.of();
+        }
+    }
+
     final class CachedApiKeyHashResult {
         final boolean success;
         final char[] hash;

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessor.java

@@ -118,21 +118,27 @@ public final class SetSecurityUserProcessor extends AbstractProcessor {
                     }
                     break;
                 case API_KEY:
-                    final String apiKey = "api_key";
-                    final Object existingApiKeyField = userObject.get(apiKey);
-                    @SuppressWarnings("unchecked")
-                    final Map<String, Object> apiKeyField =
-                        existingApiKeyField instanceof Map ? (Map<String, Object>) existingApiKeyField : new HashMap<>();
-                    Object apiKeyName = authentication.getMetadata().get(ApiKeyService.API_KEY_NAME_KEY);
-                    if (apiKeyName != null) {
-                        apiKeyField.put("name", apiKeyName);
-                    }
-                    Object apiKeyId = authentication.getMetadata().get(ApiKeyService.API_KEY_ID_KEY);
-                    if (apiKeyId != null) {
-                        apiKeyField.put("id", apiKeyId);
-                    }
-                    if (false == apiKeyField.isEmpty()) {
-                        userObject.put(apiKey, apiKeyField);
+                    if (Authentication.AuthenticationType.API_KEY == authentication.getAuthenticationType()) {
+                        final String apiKey = "api_key";
+                        final Object existingApiKeyField = userObject.get(apiKey);
+                        @SuppressWarnings("unchecked")
+                        final Map<String, Object> apiKeyField =
+                            existingApiKeyField instanceof Map ? (Map<String, Object>) existingApiKeyField : new HashMap<>();
+                        Object apiKeyName = authentication.getMetadata().get(ApiKeyService.API_KEY_NAME_KEY);
+                        if (apiKeyName != null) {
+                            apiKeyField.put("name", apiKeyName);
+                        }
+                        Object apiKeyId = authentication.getMetadata().get(ApiKeyService.API_KEY_ID_KEY);
+                        if (apiKeyId != null) {
+                            apiKeyField.put("id", apiKeyId);
+                        }
+                        final Map<String,Object> apiKeyMetadata = ApiKeyService.getApiKeyMetadata(authentication);
+                        if (false == apiKeyMetadata.isEmpty()) {
+                            apiKeyField.put("metadata", apiKeyMetadata);
+                        }
+                        if (false == apiKeyField.isEmpty()) {
+                            userObject.put(apiKey, apiKeyField);
+                        }
                     }
                     break;
                 case REALM:

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java

@@ -95,6 +95,8 @@ import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.AP
 import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.API_KEY_ROLE_DESCRIPTORS_KEY;
 import static org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR;
 import static org.elasticsearch.xpack.security.Security.SECURITY_CRYPTO_THREAD_POOL_NAME;
+import static org.elasticsearch.xpack.security.authc.ApiKeyService.API_KEY_METADATA_KEY;
+import static org.hamcrest.Matchers.anEmptyMap;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.emptyArray;
@@ -995,6 +997,32 @@ public class ApiKeyServiceTests extends ESTestCase {
         assertEquals(new BytesArray("{}"), apiKeyDoc.roleDescriptorsBytes);
     }
 
+    public void testGetApiKeyMetadata() throws IOException {
+        final Authentication apiKeyAuthentication = mock(Authentication.class);
+        when(apiKeyAuthentication.getAuthenticationType()).thenReturn(AuthenticationType.API_KEY);
+        final Map<String, Object> apiKeyMetadata = ApiKeyTests.randomMetadata();
+        if (apiKeyMetadata == null) {
+            when(apiKeyAuthentication.getMetadata()).thenReturn(Map.of());
+        } else {
+            final BytesReference metadataBytes = XContentTestUtils.convertToXContent(apiKeyMetadata, XContentType.JSON);
+            when(apiKeyAuthentication.getMetadata()).thenReturn(Map.of(API_KEY_METADATA_KEY, metadataBytes));
+        }
+
+        final Map<String, Object> restoredApiKeyMetadata = ApiKeyService.getApiKeyMetadata(apiKeyAuthentication);
+        if (apiKeyMetadata == null) {
+            assertThat(restoredApiKeyMetadata, anEmptyMap());
+        } else {
+            assertThat(restoredApiKeyMetadata, equalTo(apiKeyMetadata));
+        }
+
+        final Authentication authentication = mock(Authentication.class);
+        when(authentication.getAuthenticationType()).thenReturn(
+            randomValueOtherThan(AuthenticationType.API_KEY, () -> randomFrom(AuthenticationType.values())));
+        final IllegalArgumentException e =
+            expectThrows(IllegalArgumentException.class, () -> ApiKeyService.getApiKeyMetadata(authentication));
+        assertThat(e.getMessage(), containsString("authentication type must be [api_key]"));
+    }
+
     public static class Utils {
 
         private static final AuthenticationContextSerializer authenticationContextSerializer = new AuthenticationContextSerializer();
@@ -1134,11 +1162,11 @@ public class ApiKeyServiceTests extends ESTestCase {
 
     private void checkAuthApiKeyMetadata(Object metadata, AuthenticationResult authResult1) throws IOException {
         if (metadata == null) {
-            assertThat(authResult1.getMetadata().containsKey(ApiKeyService.API_KEY_METADATA_KEY), is(false));
+            assertThat(authResult1.getMetadata().containsKey(API_KEY_METADATA_KEY), is(false));
         } else {
             //noinspection unchecked
             assertThat(
-                authResult1.getMetadata().get(ApiKeyService.API_KEY_METADATA_KEY),
+                authResult1.getMetadata().get(API_KEY_METADATA_KEY),
                 equalTo(XContentTestUtils.convertToXContent((Map<String, Object>) metadata, XContentType.JSON)));
         }
     }

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessorTests.java

@@ -10,10 +10,13 @@ import org.elasticsearch.Version;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.XContentTestUtils;
 import org.elasticsearch.xpack.core.security.SecurityContext;
+import org.elasticsearch.xpack.core.security.action.ApiKeyTests;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.Authentication.AuthenticationType;
 import org.elasticsearch.xpack.core.security.authc.support.AuthenticationContextSerializer;
@@ -245,14 +248,19 @@ public class SetSecurityUserProcessorTests extends ESTestCase {
         Authentication.RealmRef realmRef = new Authentication.RealmRef(
             ApiKeyService.API_KEY_REALM_NAME, ApiKeyService.API_KEY_REALM_TYPE, "_node_name");
 
+        final Map<String, Object> authMetadata = new HashMap<>(Map.of(
+            ApiKeyService.API_KEY_ID_KEY, "api_key_id",
+            ApiKeyService.API_KEY_NAME_KEY, "api_key_name",
+            ApiKeyService.API_KEY_CREATOR_REALM_NAME, "creator_realm_name",
+            ApiKeyService.API_KEY_CREATOR_REALM_TYPE, "creator_realm_type"
+        ));
+        final Map<String, Object> apiKeyMetadata = ApiKeyTests.randomMetadata();
+        if (apiKeyMetadata != null) {
+            authMetadata.put(ApiKeyService.API_KEY_METADATA_KEY, XContentTestUtils.convertToXContent(apiKeyMetadata, XContentType.JSON));
+        }
+
         Authentication auth = new Authentication(user, realmRef, null, Version.CURRENT,
-            AuthenticationType.API_KEY,
-            Map.of(
-                ApiKeyService.API_KEY_ID_KEY, "api_key_id",
-                ApiKeyService.API_KEY_NAME_KEY, "api_key_name",
-                ApiKeyService.API_KEY_CREATOR_REALM_NAME, "creator_realm_name",
-                ApiKeyService.API_KEY_CREATOR_REALM_TYPE, "creator_realm_type"
-            ));
+            AuthenticationType.API_KEY, authMetadata);
         auth.writeToContext(threadContext);
 
         IngestDocument ingestDocument = new IngestDocument(new HashMap<>(), new HashMap<>());
@@ -262,8 +270,14 @@ public class SetSecurityUserProcessorTests extends ESTestCase {
 
         Map<String, Object> result = ingestDocument.getFieldValue("_field", Map.class);
         assertThat(result.size(), equalTo(4));
-        assertThat(((Map) result.get("api_key")).get("name"), equalTo("api_key_name"));
-        assertThat(((Map) result.get("api_key")).get("id"), equalTo("api_key_id"));
+        final Map<String, Object> apiKeyMap = (Map<String, Object>) result.get("api_key");
+        assertThat(apiKeyMap.get("name"), equalTo("api_key_name"));
+        assertThat(apiKeyMap.get("id"), equalTo("api_key_id"));
+        if (apiKeyMetadata == null || apiKeyMetadata.isEmpty()) {
+            assertNull(apiKeyMap.get("metadata"));
+        } else {
+            assertThat(apiKeyMap.get("metadata"), equalTo(apiKeyMetadata));
+        }
         assertThat(((Map) result.get("realm")).get("name"), equalTo("creator_realm_name"));
         assertThat(((Map) result.get("realm")).get("type"), equalTo("creator_realm_type"));
         assertThat(result.get("authentication_type"), equalTo("API_KEY"));
@@ -274,14 +288,19 @@ public class SetSecurityUserProcessorTests extends ESTestCase {
         Authentication.RealmRef realmRef = new Authentication.RealmRef(
             ApiKeyService.API_KEY_REALM_NAME, ApiKeyService.API_KEY_REALM_TYPE, "_node_name");
 
+        final Map<String, Object> authMetadata = new HashMap<>(Map.of(
+            ApiKeyService.API_KEY_ID_KEY, "api_key_id",
+            ApiKeyService.API_KEY_NAME_KEY, "api_key_name",
+            ApiKeyService.API_KEY_CREATOR_REALM_NAME, "creator_realm_name",
+            ApiKeyService.API_KEY_CREATOR_REALM_TYPE, "creator_realm_type"
+        ));
+        final Map<String, Object> apiKeyMetadata = ApiKeyTests.randomMetadata();
+        if (apiKeyMetadata != null) {
+            authMetadata.put(ApiKeyService.API_KEY_METADATA_KEY, XContentTestUtils.convertToXContent(apiKeyMetadata, XContentType.JSON));
+        }
+
         Authentication auth = new Authentication(user, realmRef, null, Version.CURRENT,
-            AuthenticationType.API_KEY,
-            Map.of(
-                ApiKeyService.API_KEY_ID_KEY, "api_key_id",
-                ApiKeyService.API_KEY_NAME_KEY, "api_key_name",
-                ApiKeyService.API_KEY_CREATOR_REALM_NAME, "creator_realm_name",
-                ApiKeyService.API_KEY_CREATOR_REALM_TYPE, "creator_realm_type"
-            ));
+            AuthenticationType.API_KEY, authMetadata);
         auth.writeToContext(threadContext);
 
         IngestDocument ingestDocument = new IngestDocument(IngestDocument.deepCopyMap(Map.of(
@@ -297,7 +316,7 @@ public class SetSecurityUserProcessorTests extends ESTestCase {
         assertThat(((Map) result.get("realm")).get("id"), equalTo(7));
     }
 
-    public void testWillSetRunAsRealmForNonApiAuth() throws Exception {
+    public void testWillSetRunAsRealmForNonApiKeyAuth() throws Exception {
         User user = new User(randomAlphaOfLengthBetween(4, 12), null, null);
         Authentication.RealmRef authRealmRef = new Authentication.RealmRef(
             randomAlphaOfLengthBetween(4, 12), randomAlphaOfLengthBetween(4, 12), randomAlphaOfLengthBetween(4, 12));

x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/set_security_user/20_api_key.yml

@@ -0,0 +1,127 @@
+---
+setup:
+  - skip:
+      features: headers
+
+  - do:
+      cluster.health:
+        wait_for_status: yellow
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "processors": [
+              {
+                "set_security_user" : {
+                  "field" : "user"
+                }
+              }
+            ]
+          }
+
+---
+teardown:
+  - do:
+      ingest.delete_pipeline:
+        id: "my_pipeline"
+        ignore: 404
+
+---
+"Test set security user ingest processor works for API keys":
+  - do:
+      security.create_api_key:
+        body:  >
+          {
+            "name": "with-metadata",
+            "metadata": {
+              "string": "hello",
+              "number": 42,
+              "complex": { "foo": "bar", "values": [1, 3, 5] }
+            }
+          }
+  - set: { id: id_with_metadata }
+  - transform_and_set: { login_creds: "#base64EncodeCredentials(id,api_key)" }
+
+  - do:
+      headers:
+        Authorization: ApiKey ${login_creds}
+      index:
+        index: index
+        id: 1
+        pipeline: "my_pipeline"
+        body: >
+          {
+            "message": "should have api key metadata"
+          }
+
+  - do:
+      security.create_api_key:
+        body:  >
+          {
+            "name": "no-metadata"
+          }
+  - set: { id: id_no_metadata }
+  - transform_and_set: { login_creds: "#base64EncodeCredentials(id,api_key)" }
+
+  - do:
+      headers:
+        Authorization: ApiKey ${login_creds}
+      index:
+        index: index
+        id: 2
+        pipeline: "my_pipeline"
+        body: >
+          {
+            "message": "should not have api key metadata"
+          }
+
+  - do:
+      security.create_api_key:
+        body:  >
+          {
+            "name": "empty-metadata",
+            "metadata": { }
+          }
+  - set: { id: id_empty_metadata }
+  - transform_and_set: { login_creds: "#base64EncodeCredentials(id,api_key)" }
+
+  - do:
+      headers:
+        Authorization: ApiKey ${login_creds}
+      index:
+        index: index
+        id: 3
+        pipeline: "my_pipeline"
+        body: >
+          {
+            "message": "should not have api key metadata either"
+          }
+
+  - do:
+      indices.refresh:
+        index: index
+
+  - do:
+      get:
+        index: index
+        id: 1
+  - match: { _source.user.api_key.name: "with-metadata" }
+  - match: { _source.user.api_key.id: $id_with_metadata }
+  - match: { _source.user.api_key.metadata: { "string": "hello", "number": 42, "complex": {"foo": "bar", "values": [1, 3, 5]} } }
+
+  - do:
+      get:
+        index: index
+        id: 2
+  - match: { _source.user.api_key.name: "no-metadata" }
+  - match: { _source.user.api_key.id: $id_no_metadata }
+  - is_false: _source.user.api_key.metadata
+
+  - do:
+      get:
+        index: index
+        id: 3
+  - match: { _source.user.api_key.name: "empty-metadata" }
+  - match: { _source.user.api_key.id: $id_empty_metadata }
+  - is_false: _source.user.api_key.metadata