Show assigned role descriptors in Get/QueryApiKey response (#89166)

This PR adds a new `role_descriptors` field in the API key entity
returned by both GetApiKey and QueryApiKey APIs. The field value is the
map of the role descriptors that are assigned to an API key when
creating or updating the key. If the key has no assigned role
descriptors, i.e. it inherits the owner user's privileges, an empty
object is returned in place.

Relates: #89058

docs/changelog/89166.yaml

@@ -0,0 +1,5 @@
+pr: 89166
+summary: Show assigned role descriptors in Get/QueryApiKey response
+area: Security
+type: enhancement
+issues: []

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/ApiKey.java

@@ -17,9 +17,11 @@ import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 
 import java.io.IOException;
 import java.time.Instant;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
@@ -39,6 +41,8 @@ public final class ApiKey implements ToXContentObject, Writeable {
     private final String username;
     private final String realm;
     private final Map<String, Object> metadata;
+    @Nullable
+    private final List<RoleDescriptor> roleDescriptors;
 
     public ApiKey(
         String name,
@@ -48,7 +52,8 @@ public final class ApiKey implements ToXContentObject, Writeable {
         boolean invalidated,
         String username,
         String realm,
-        @Nullable Map<String, Object> metadata
+        @Nullable Map<String, Object> metadata,
+        @Nullable List<RoleDescriptor> roleDescriptors
     ) {
         this.name = name;
         this.id = id;
@@ -61,6 +66,7 @@ public final class ApiKey implements ToXContentObject, Writeable {
         this.username = username;
         this.realm = realm;
         this.metadata = metadata == null ? Map.of() : metadata;
+        this.roleDescriptors = roleDescriptors;
     }
 
     public ApiKey(StreamInput in) throws IOException {
@@ -80,6 +86,12 @@ public final class ApiKey implements ToXContentObject, Writeable {
         } else {
             this.metadata = Map.of();
         }
+        if (in.getVersion().onOrAfter(Version.V_8_5_0)) {
+            final List<RoleDescriptor> roleDescriptors = in.readOptionalList(RoleDescriptor::new);
+            this.roleDescriptors = roleDescriptors != null ? List.copyOf(roleDescriptors) : null;
+        } else {
+            this.roleDescriptors = null;
+        }
     }
 
     public String getId() {
@@ -114,6 +126,10 @@ public final class ApiKey implements ToXContentObject, Writeable {
         return metadata;
     }
 
+    public List<RoleDescriptor> getRoleDescriptors() {
+        return roleDescriptors;
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
@@ -130,6 +146,13 @@ public final class ApiKey implements ToXContentObject, Writeable {
             .field("username", username)
             .field("realm", realm)
             .field("metadata", (metadata == null ? Map.of() : metadata));
+        if (roleDescriptors != null) {
+            builder.startObject("role_descriptors");
+            for (var roleDescriptor : roleDescriptors) {
+                builder.field(roleDescriptor.getName(), roleDescriptor);
+            }
+            builder.endObject();
+        }
         return builder;
     }
 
@@ -149,11 +172,14 @@ public final class ApiKey implements ToXContentObject, Writeable {
         if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
             out.writeGenericMap(metadata);
         }
+        if (out.getVersion().onOrAfter(Version.V_8_5_0)) {
+            out.writeOptionalCollection(roleDescriptors);
+        }
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(name, id, creation, expiration, invalidated, username, realm, metadata);
+        return Objects.hash(name, id, creation, expiration, invalidated, username, realm, metadata, roleDescriptors);
     }
 
     @Override
@@ -175,7 +201,8 @@ public final class ApiKey implements ToXContentObject, Writeable {
             && Objects.equals(invalidated, other.invalidated)
             && Objects.equals(username, other.username)
             && Objects.equals(realm, other.realm)
-            && Objects.equals(metadata, other.metadata);
+            && Objects.equals(metadata, other.metadata)
+            && Objects.equals(roleDescriptors, other.roleDescriptors);
     }
 
     @SuppressWarnings("unchecked")
@@ -188,7 +215,8 @@ public final class ApiKey implements ToXContentObject, Writeable {
             (Boolean) args[4],
             (String) args[5],
             (String) args[6],
-            (args[7] == null) ? null : (Map<String, Object>) args[7]
+            (args[7] == null) ? null : (Map<String, Object>) args[7],
+            (List<RoleDescriptor>) args[8]
         );
     });
     static {
@@ -200,6 +228,10 @@ public final class ApiKey implements ToXContentObject, Writeable {
         PARSER.declareString(constructorArg(), new ParseField("username"));
         PARSER.declareString(constructorArg(), new ParseField("realm"));
         PARSER.declareObject(optionalConstructorArg(), (p, c) -> p.map(), new ParseField("metadata"));
+        PARSER.declareNamedObjects(optionalConstructorArg(), (p, c, n) -> {
+            p.nextToken();
+            return RoleDescriptor.parse(n, p, false);
+        }, new ParseField("role_descriptors"));
     }
 
     public static ApiKey fromXContent(XContentParser parser) throws IOException {
@@ -224,6 +256,8 @@ public final class ApiKey implements ToXContentObject, Writeable {
             + realm
             + ", metadata="
             + metadata
+            + ", role_descriptors="
+            + roleDescriptors
             + "]";
     }
 

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/ApiKeyTests.java

@@ -10,9 +10,11 @@ package org.elasticsearch.xpack.core.security.action.apikey;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.XContentTestUtils;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 
 import java.io.IOException;
 import java.time.Instant;
@@ -20,8 +22,11 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
+import static org.elasticsearch.xpack.core.security.authz.RoleDescriptorTests.randomUniquelyNamedRoleDescriptors;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
 
 public class ApiKeyTests extends ESTestCase {
@@ -38,8 +43,9 @@ public class ApiKeyTests extends ESTestCase {
         final String username = randomAlphaOfLengthBetween(4, 10);
         final String realmName = randomAlphaOfLengthBetween(3, 8);
         final Map<String, Object> metadata = randomMetadata();
+        final List<RoleDescriptor> roleDescriptors = randomBoolean() ? null : randomUniquelyNamedRoleDescriptors(0, 3);
 
-        final ApiKey apiKey = new ApiKey(name, id, creation, expiration, invalidated, username, realmName, metadata);
+        final ApiKey apiKey = new ApiKey(name, id, creation, expiration, invalidated, username, realmName, metadata, roleDescriptors);
         // The metadata will never be null because the constructor convert it to empty map if a null is passed in
         assertThat(apiKey.getMetadata(), notNullValue());
 
@@ -59,6 +65,18 @@ public class ApiKeyTests extends ESTestCase {
         assertThat(map.get("username"), equalTo(username));
         assertThat(map.get("realm"), equalTo(realmName));
         assertThat(map.get("metadata"), equalTo(Objects.requireNonNullElseGet(metadata, Map::of)));
+
+        if (roleDescriptors == null) {
+            assertThat(map, not(hasKey("role_descriptors")));
+        } else {
+            @SuppressWarnings("unchecked")
+            final Map<String, Object> rdMap = (Map<String, Object>) map.get("role_descriptors");
+            assertThat(rdMap.size(), equalTo(roleDescriptors.size()));
+            for (var roleDescriptor : roleDescriptors) {
+                assertThat(rdMap, hasKey(roleDescriptor.getName()));
+                assertThat(XContentTestUtils.convertToMap(roleDescriptor), equalTo(rdMap.get(roleDescriptor.getName())));
+            }
+        }
     }
 
     @SuppressWarnings("unchecked")

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/GetApiKeyResponseTests.java

@@ -9,19 +9,26 @@ package org.elasticsearch.xpack.core.security.action.apikey;
 
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivilege;
+import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivileges;
 
 import java.io.IOException;
 import java.time.Instant;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.xpack.core.security.authz.RoleDescriptorTests.randomUniquelyNamedRoleDescriptors;
 import static org.hamcrest.Matchers.equalTo;
 
 public class GetApiKeyResponseTests extends ESTestCase {
@@ -37,12 +44,29 @@ public class GetApiKeyResponseTests extends ESTestCase {
             false,
             randomAlphaOfLength(4),
             randomAlphaOfLength(5),
-            randomBoolean() ? null : Map.of(randomAlphaOfLengthBetween(3, 8), randomAlphaOfLengthBetween(3, 8))
+            randomBoolean() ? null : Map.of(randomAlphaOfLengthBetween(3, 8), randomAlphaOfLengthBetween(3, 8)),
+            randomBoolean() ? null : randomUniquelyNamedRoleDescriptors(0, 3)
         );
         GetApiKeyResponse response = new GetApiKeyResponse(Collections.singletonList(apiKeyInfo));
+
+        final NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(
+            List.of(
+                new NamedWriteableRegistry.Entry(
+                    ConfigurableClusterPrivilege.class,
+                    ConfigurableClusterPrivileges.ManageApplicationPrivileges.WRITEABLE_NAME,
+                    ConfigurableClusterPrivileges.ManageApplicationPrivileges::createFrom
+                ),
+                new NamedWriteableRegistry.Entry(
+                    ConfigurableClusterPrivilege.class,
+                    ConfigurableClusterPrivileges.WriteProfileDataPrivileges.WRITEABLE_NAME,
+                    ConfigurableClusterPrivileges.WriteProfileDataPrivileges::createFrom
+                )
+            )
+        );
+
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             response.writeTo(output);
-            try (StreamInput input = output.bytes().streamInput()) {
+            try (StreamInput input = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
                 GetApiKeyResponse serialized = new GetApiKeyResponse(input);
                 assertThat(serialized.getApiKeyInfos(), equalTo(response.getApiKeyInfos()));
             }
@@ -50,6 +74,16 @@ public class GetApiKeyResponseTests extends ESTestCase {
     }
 
     public void testToXContent() throws IOException {
+        final List<RoleDescriptor> roleDescriptors = List.of(
+            new RoleDescriptor(
+                "rd_42",
+                new String[] { "monitor" },
+                new RoleDescriptor.IndicesPrivileges[] {
+                    RoleDescriptor.IndicesPrivileges.builder().indices("index").privileges("read").build() },
+                new String[] { "foo" }
+            )
+        );
+
         ApiKey apiKeyInfo1 = createApiKeyInfo(
             "name1",
             "id-1",
@@ -58,6 +92,7 @@ public class GetApiKeyResponseTests extends ESTestCase {
             false,
             "user-a",
             "realm-x",
+            null,
             null
         );
         ApiKey apiKeyInfo2 = createApiKeyInfo(
@@ -68,7 +103,8 @@ public class GetApiKeyResponseTests extends ESTestCase {
             true,
             "user-b",
             "realm-y",
-            Map.of()
+            Map.of(),
+            List.of()
         );
         ApiKey apiKeyInfo3 = createApiKeyInfo(
             null,
@@ -78,7 +114,8 @@ public class GetApiKeyResponseTests extends ESTestCase {
             true,
             "user-c",
             "realm-z",
-            Map.of("foo", "bar")
+            Map.of("foo", "bar"),
+            roleDescriptors
         );
         GetApiKeyResponse response = new GetApiKeyResponse(Arrays.asList(apiKeyInfo1, apiKeyInfo2, apiKeyInfo3));
         XContentBuilder builder = XContentFactory.jsonBuilder();
@@ -104,7 +141,8 @@ public class GetApiKeyResponseTests extends ESTestCase {
                   "invalidated": true,
                   "username": "user-b",
                   "realm": "realm-y",
-                  "metadata": {}
+                  "metadata": {},
+                  "role_descriptors": {}
                 },
                 {
                   "id": "id-3",
@@ -115,6 +153,32 @@ public class GetApiKeyResponseTests extends ESTestCase {
                   "realm": "realm-z",
                   "metadata": {
                     "foo": "bar"
+                  },
+                  "role_descriptors": {
+                    "rd_42": {
+                      "cluster": [
+                        "monitor"
+                      ],
+                      "indices": [
+                        {
+                          "names": [
+                            "index"
+                          ],
+                          "privileges": [
+                            "read"
+                          ],
+                          "allow_restricted_indices": false
+                        }
+                      ],
+                      "applications": [],
+                      "run_as": [
+                        "foo"
+                      ],
+                      "metadata": {},
+                      "transient_metadata": {
+                        "enabled": true
+                      }
+                    }
                   }
                 }
               ]
@@ -129,8 +193,9 @@ public class GetApiKeyResponseTests extends ESTestCase {
         boolean invalidated,
         String username,
         String realm,
-        Map<String, Object> metadata
+        Map<String, Object> metadata,
+        List<RoleDescriptor> roleDescriptors
     ) {
-        return new ApiKey(name, id, creation, expiration, invalidated, username, realm, metadata);
+        return new ApiKey(name, id, creation, expiration, invalidated, username, realm, metadata, roleDescriptors);
     }
 }

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/QueryApiKeyResponseTests.java

@@ -7,8 +7,12 @@
 
 package org.elasticsearch.xpack.core.security.action.apikey;
 
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivilege;
+import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivileges;
 
 import java.io.IOException;
 import java.time.Instant;
@@ -18,6 +22,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.xpack.core.security.authz.RoleDescriptorTests.randomUniquelyNamedRoleDescriptors;
+
 public class QueryApiKeyResponseTests extends AbstractWireSerializingTestCase<QueryApiKeyResponse> {
 
     @Override
@@ -58,6 +64,24 @@ public class QueryApiKeyResponseTests extends AbstractWireSerializingTestCase<Qu
         }
     }
 
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(
+            List.of(
+                new NamedWriteableRegistry.Entry(
+                    ConfigurableClusterPrivilege.class,
+                    ConfigurableClusterPrivileges.ManageApplicationPrivileges.WRITEABLE_NAME,
+                    ConfigurableClusterPrivileges.ManageApplicationPrivileges::createFrom
+                ),
+                new NamedWriteableRegistry.Entry(
+                    ConfigurableClusterPrivilege.class,
+                    ConfigurableClusterPrivileges.WriteProfileDataPrivileges.WRITEABLE_NAME,
+                    ConfigurableClusterPrivileges.WriteProfileDataPrivileges::createFrom
+                )
+            )
+        );
+    }
+
     private QueryApiKeyResponse.Item randomItem() {
         return new QueryApiKeyResponse.Item(randomApiKeyInfo(), randomSortValues());
     }
@@ -70,7 +94,8 @@ public class QueryApiKeyResponseTests extends AbstractWireSerializingTestCase<Qu
         final Instant creation = Instant.ofEpochMilli(randomMillisUpToYear9999());
         final Instant expiration = randomBoolean() ? Instant.ofEpochMilli(randomMillisUpToYear9999()) : null;
         final Map<String, Object> metadata = ApiKeyTests.randomMetadata();
-        return new ApiKey(name, id, creation, expiration, false, username, realm_name, metadata);
+        final List<RoleDescriptor> roleDescriptors = randomFrom(randomUniquelyNamedRoleDescriptors(0, 3), null);
+        return new ApiKey(name, id, creation, expiration, false, username, realm_name, metadata, roleDescriptors);
     }
 
     private Object[] randomSortValues() {

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/RoleDescriptorTests.java → x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptorTests.java

@@ -4,7 +4,7 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-package org.elasticsearch.xpack.security.authz;
+package org.elasticsearch.xpack.core.security.authz;
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.Version;
@@ -24,7 +24,6 @@ import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.core.XPackClientPlugin;
-import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.ApplicationResourcePrivileges;
 import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilegeResolver;
 import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivilege;
@@ -626,6 +625,13 @@ public class RoleDescriptorTests extends ESTestCase {
         }
     }
 
+    public static List<RoleDescriptor> randomUniquelyNamedRoleDescriptors(int minSize, int maxSize) {
+        return randomValueOtherThanMany(
+            roleDescriptors -> roleDescriptors.stream().map(RoleDescriptor::getName).distinct().count() != roleDescriptors.size(),
+            () -> randomList(minSize, maxSize, () -> randomRoleDescriptor(false))
+        );
+    }
+
     public static RoleDescriptor randomRoleDescriptor() {
         return randomRoleDescriptor(true);
     }

x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java

@@ -20,6 +20,7 @@ import org.elasticsearch.xpack.core.security.action.apikey.ApiKey;
 import org.elasticsearch.xpack.core.security.action.apikey.GetApiKeyResponse;
 import org.elasticsearch.xpack.core.security.action.apikey.GrantApiKeyAction;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.security.SecurityOnTrialLicenseRestTestCase;
 import org.junit.After;
 import org.junit.Before;
@@ -32,6 +33,7 @@ import java.util.Map;
 import java.util.Set;
 
 import static org.elasticsearch.xpack.core.security.authc.AuthenticationServiceField.RUN_AS_USER_HEADER;
+import static org.hamcrest.Matchers.anEmptyMap;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
@@ -79,6 +81,125 @@ public class ApiKeyRestIT extends SecurityOnTrialLicenseRestTestCase {
         invalidateApiKeysForUser(MANAGE_OWN_API_KEY_USER);
     }
 
+    @SuppressWarnings("unchecked")
+    public void testGetApiKeyRoleDescriptors() throws IOException {
+        // First key without assigned role descriptors, i.e. it inherits owner user's permission
+        // This can be achieved by either omitting the role_descriptors field in the request or
+        // explicitly set it to an empty object
+        final Request createApiKeyRequest1 = new Request("POST", "_security/api_key");
+        if (randomBoolean()) {
+            createApiKeyRequest1.setJsonEntity("""
+                {
+                  "name": "k1"
+                }""");
+        } else {
+            createApiKeyRequest1.setJsonEntity("""
+                {
+                  "name": "k1",
+                  "role_descriptors": { }
+                }""");
+        }
+        assertOK(adminClient().performRequest(createApiKeyRequest1));
+
+        // Second key with a single assigned role descriptor
+        final Request createApiKeyRequest2 = new Request("POST", "_security/api_key");
+        createApiKeyRequest2.setJsonEntity("""
+            {
+              "name": "k2",
+                "role_descriptors": {
+                  "x": {
+                    "cluster": [
+                      "monitor"
+                    ]
+                  }
+                }
+            }""");
+        assertOK(adminClient().performRequest(createApiKeyRequest2));
+
+        // Third key with two assigned role descriptors
+        final Request createApiKeyRequest3 = new Request("POST", "_security/api_key");
+        createApiKeyRequest3.setJsonEntity("""
+            {
+              "name": "k3",
+                "role_descriptors": {
+                  "x": {
+                    "cluster": [
+                      "monitor"
+                    ]
+                  },
+                  "y": {
+                    "indices": [
+                      {
+                        "names": [
+                          "index"
+                        ],
+                        "privileges": [
+                          "read"
+                        ]
+                      }
+                    ]
+                  }
+                }
+            }""");
+        assertOK(adminClient().performRequest(createApiKeyRequest3));
+
+        // Role descriptors are returned by both get and query api key calls
+        final List<Map<String, Object>> apiKeyMaps;
+        if (randomBoolean()) {
+            final Request getApiKeyRequest = new Request("GET", "_security/api_key");
+            final Response getApiKeyResponse = adminClient().performRequest(getApiKeyRequest);
+            assertOK(getApiKeyResponse);
+            apiKeyMaps = (List<Map<String, Object>>) responseAsMap(getApiKeyResponse).get("api_keys");
+        } else {
+            final Request queryApiKeyRequest = new Request("POST", "_security/_query/api_key");
+            final Response queryApiKeyResponse = adminClient().performRequest(queryApiKeyRequest);
+            assertOK(queryApiKeyResponse);
+            apiKeyMaps = (List<Map<String, Object>>) responseAsMap(queryApiKeyResponse).get("api_keys");
+        }
+        assertThat(apiKeyMaps.size(), equalTo(3));
+
+        for (Map<String, Object> apiKeyMap : apiKeyMaps) {
+            final String name = (String) apiKeyMap.get("name");
+            @SuppressWarnings("unchecked")
+            final var roleDescriptors = (Map<String, Object>) apiKeyMap.get("role_descriptors");
+            switch (name) {
+                case "k1" -> {
+                    assertThat(roleDescriptors, anEmptyMap());
+                }
+                case "k2" -> {
+                    assertThat(
+                        roleDescriptors,
+                        equalTo(
+                            Map.of("x", XContentTestUtils.convertToMap(new RoleDescriptor("x", new String[] { "monitor" }, null, null)))
+                        )
+                    );
+                }
+                case "k3" -> {
+                    assertThat(
+                        roleDescriptors,
+                        equalTo(
+                            Map.of(
+                                "x",
+                                XContentTestUtils.convertToMap(new RoleDescriptor("x", new String[] { "monitor" }, null, null)),
+                                "y",
+                                XContentTestUtils.convertToMap(
+                                    new RoleDescriptor(
+                                        "y",
+                                        null,
+                                        new RoleDescriptor.IndicesPrivileges[] {
+                                            RoleDescriptor.IndicesPrivileges.builder().indices("index").privileges("read").build() },
+                                        null
+                                    )
+                                )
+                            )
+                        )
+                    );
+                }
+                default -> throw new IllegalStateException("unknown api key name [" + name + "]");
+            }
+        }
+    }
+
     @SuppressWarnings({ "unchecked" })
     public void testAuthenticateResponseApiKey() throws IOException {
         final String expectedApiKeyName = "my-api-key-name";

x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java

@@ -84,10 +84,10 @@ import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.RealmDomain;
 import org.elasticsearch.xpack.core.security.authc.file.FileRealmSettings;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptorTests;
 import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilegeResolver;
 import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
 import org.elasticsearch.xpack.core.security.user.User;
-import org.elasticsearch.xpack.security.authz.RoleDescriptorTests;
 import org.elasticsearch.xpack.security.transport.filter.IPFilter;
 import org.junit.After;
 import org.junit.Before;
@@ -649,13 +649,14 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
             2,
             responses,
             tuple.v2(),
+            List.of(DEFAULT_API_KEY_ROLE_DESCRIPTOR),
             response,
             Collections.singleton(responses.get(0).getId()),
             Collections.singletonList(responses.get(1).getId())
         );
     }
 
-    public void testGetApiKeysForRealm() throws InterruptedException, ExecutionException {
+    public void testGetApiKeysForRealm() throws InterruptedException, ExecutionException, IOException {
         int noOfApiKeys = randomIntBetween(3, 5);
         final Tuple<List<CreateApiKeyResponse>, List<Map<String, Object>>> tuple = createApiKeys(noOfApiKeys, null);
         List<CreateApiKeyResponse> responses = tuple.v1();
@@ -686,7 +687,15 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
         PlainActionFuture<GetApiKeyResponse> listener = new PlainActionFuture<>();
         client.execute(GetApiKeyAction.INSTANCE, GetApiKeyRequest.usingRealmName("file"), listener);
         GetApiKeyResponse response = listener.get();
-        verifyGetResponse(noOfApiKeys, responses, tuple.v2(), response, expectedValidKeyIds, invalidatedApiKeyIds);
+        verifyGetResponse(
+            noOfApiKeys,
+            responses,
+            tuple.v2(),
+            List.of(DEFAULT_API_KEY_ROLE_DESCRIPTOR),
+            response,
+            expectedValidKeyIds,
+            invalidatedApiKeyIds
+        );
     }
 
     public void testGetApiKeysForUser() throws Exception {
@@ -703,13 +712,14 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
             noOfApiKeys,
             responses,
             tuple.v2(),
+            List.of(DEFAULT_API_KEY_ROLE_DESCRIPTOR),
             response,
             responses.stream().map(o -> o.getId()).collect(Collectors.toSet()),
             null
         );
     }
 
-    public void testGetApiKeysForRealmAndUser() throws InterruptedException, ExecutionException {
+    public void testGetApiKeysForRealmAndUser() throws InterruptedException, ExecutionException, IOException {
         final Tuple<List<CreateApiKeyResponse>, List<Map<String, Object>>> tuple = createApiKeys(1, null);
         List<CreateApiKeyResponse> responses = tuple.v1();
         Client client = client().filterWithHeader(
@@ -718,10 +728,18 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
         PlainActionFuture<GetApiKeyResponse> listener = new PlainActionFuture<>();
         client.execute(GetApiKeyAction.INSTANCE, GetApiKeyRequest.usingRealmAndUserName("file", ES_TEST_ROOT_USER), listener);
         GetApiKeyResponse response = listener.get();
-        verifyGetResponse(1, responses, tuple.v2(), response, Collections.singleton(responses.get(0).getId()), null);
+        verifyGetResponse(
+            1,
+            responses,
+            tuple.v2(),
+            List.of(DEFAULT_API_KEY_ROLE_DESCRIPTOR),
+            response,
+            Collections.singleton(responses.get(0).getId()),
+            null
+        );
     }
 
-    public void testGetApiKeysForApiKeyId() throws InterruptedException, ExecutionException {
+    public void testGetApiKeysForApiKeyId() throws InterruptedException, ExecutionException, IOException {
         final Tuple<List<CreateApiKeyResponse>, List<Map<String, Object>>> tuple = createApiKeys(1, null);
         List<CreateApiKeyResponse> responses = tuple.v1();
         Client client = client().filterWithHeader(
@@ -730,10 +748,18 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
         PlainActionFuture<GetApiKeyResponse> listener = new PlainActionFuture<>();
         client.execute(GetApiKeyAction.INSTANCE, GetApiKeyRequest.usingApiKeyId(responses.get(0).getId(), false), listener);
         GetApiKeyResponse response = listener.get();
-        verifyGetResponse(1, responses, tuple.v2(), response, Collections.singleton(responses.get(0).getId()), null);
+        verifyGetResponse(
+            1,
+            responses,
+            tuple.v2(),
+            List.of(DEFAULT_API_KEY_ROLE_DESCRIPTOR),
+            response,
+            Collections.singleton(responses.get(0).getId()),
+            null
+        );
     }
 
-    public void testGetApiKeysForApiKeyName() throws InterruptedException, ExecutionException {
+    public void testGetApiKeysForApiKeyName() throws InterruptedException, ExecutionException, IOException {
         final Map<String, String> headers = Collections.singletonMap(
             "Authorization",
             basicAuthHeaderValue(ES_TEST_ROOT_USER, TEST_PASSWORD_SECURE_STRING)
@@ -757,7 +783,16 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
         List<CreateApiKeyResponse> responses = randomFrom(createApiKeyResponses1, createApiKeyResponses2);
         List<Map<String, Object>> metadatas = responses == createApiKeyResponses1 ? tuple1.v2() : tuple2.v2();
         client.execute(GetApiKeyAction.INSTANCE, GetApiKeyRequest.usingApiKeyName(responses.get(0).getName(), false), listener);
-        verifyGetResponse(1, responses, metadatas, listener.get(), Collections.singleton(responses.get(0).getId()), null);
+        // role descriptors are the same between randomization
+        verifyGetResponse(
+            1,
+            responses,
+            metadatas,
+            List.of(DEFAULT_API_KEY_ROLE_DESCRIPTOR),
+            listener.get(),
+            Collections.singleton(responses.get(0).getId()),
+            null
+        );
 
         PlainActionFuture<GetApiKeyResponse> listener2 = new PlainActionFuture<>();
         client.execute(GetApiKeyAction.INSTANCE, GetApiKeyRequest.usingApiKeyName("test-key*", false), listener2);
@@ -765,10 +800,15 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
             noOfApiKeys,
             createApiKeyResponses1,
             tuple1.v2(),
+            List.of(DEFAULT_API_KEY_ROLE_DESCRIPTOR),
             listener2.get(),
             createApiKeyResponses1.stream().map(CreateApiKeyResponse::getId).collect(Collectors.toSet()),
             null
         );
+        expectAttributesForApiKeys(
+            createApiKeyResponses1.stream().map(CreateApiKeyResponse::getId).toList(),
+            Map.of(ApiKeyAttribute.ASSIGNED_ROLE_DESCRIPTORS, List.of(DEFAULT_API_KEY_ROLE_DESCRIPTOR))
+        );
 
         PlainActionFuture<GetApiKeyResponse> listener3 = new PlainActionFuture<>();
         client.execute(GetApiKeyAction.INSTANCE, GetApiKeyRequest.usingApiKeyName("*", false), listener3);
@@ -778,6 +818,7 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
             2 * noOfApiKeys,
             responses,
             metadatas,
+            List.of(DEFAULT_API_KEY_ROLE_DESCRIPTOR),
             listener3.get(),
             responses.stream().map(CreateApiKeyResponse::getId).collect(Collectors.toSet()),
             null
@@ -785,7 +826,7 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
 
         PlainActionFuture<GetApiKeyResponse> listener4 = new PlainActionFuture<>();
         client.execute(GetApiKeyAction.INSTANCE, GetApiKeyRequest.usingApiKeyName("does-not-exist*", false), listener4);
-        verifyGetResponse(0, Collections.emptyList(), null, listener4.get(), Collections.emptySet(), null);
+        verifyGetResponse(0, Collections.emptyList(), null, List.of(), listener4.get(), Collections.emptySet(), null);
 
         PlainActionFuture<GetApiKeyResponse> listener5 = new PlainActionFuture<>();
         client.execute(GetApiKeyAction.INSTANCE, GetApiKeyRequest.usingApiKeyName("another-test-key*", false), listener5);
@@ -793,6 +834,7 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
             noOfApiKeys,
             createApiKeyResponses2,
             tuple2.v2(),
+            List.of(DEFAULT_API_KEY_ROLE_DESCRIPTOR),
             listener5.get(),
             createApiKeyResponses2.stream().map(CreateApiKeyResponse::getId).collect(Collectors.toSet()),
             null
@@ -823,6 +865,7 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
             noOfApiKeysForUserWithManageApiKeyRole,
             userWithManageApiKeyRoleApiKeys,
             tuple.v2(),
+            List.of(DEFAULT_API_KEY_ROLE_DESCRIPTOR),
             response,
             userWithManageApiKeyRoleApiKeys.stream().map(o -> o.getId()).collect(Collectors.toSet()),
             null
@@ -850,6 +893,7 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
             noOfApiKeysForUserWithManageApiKeyRole,
             userWithManageOwnApiKeyRoleApiKeys,
             tuple.v2(),
+            List.of(DEFAULT_API_KEY_ROLE_DESCRIPTOR),
             response,
             userWithManageOwnApiKeyRoleApiKeys.stream().map(o -> o.getId()).collect(Collectors.toSet()),
             null
@@ -881,6 +925,7 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
             noOfApiKeysForUserWithManageApiKeyRole,
             userWithManageOwnApiKeyRoleApiKeys,
             tuple.v2(),
+            List.of(DEFAULT_API_KEY_ROLE_DESCRIPTOR),
             response,
             userWithManageOwnApiKeyRoleApiKeys.stream().map(o -> o.getId()).collect(Collectors.toSet()),
             null
@@ -956,6 +1001,7 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
             totalApiKeys,
             allApiKeys,
             metadatas,
+            List.of(DEFAULT_API_KEY_ROLE_DESCRIPTOR),
             response,
             allApiKeys.stream().map(o -> o.getId()).collect(Collectors.toSet()),
             null
@@ -1098,7 +1144,15 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
         PlainActionFuture<GetApiKeyResponse> listener = new PlainActionFuture<>();
         client.execute(GetApiKeyAction.INSTANCE, GetApiKeyRequest.usingApiKeyId(responses.get(0).getId(), randomBoolean()), listener);
         GetApiKeyResponse response = listener.get();
-        verifyGetResponse(1, responses, tuple.v2(), response, Collections.singleton(responses.get(0).getId()), null);
+        verifyGetResponse(
+            1,
+            responses,
+            tuple.v2(),
+            List.of(new RoleDescriptor(DEFAULT_API_KEY_ROLE_DESCRIPTOR.getName(), Strings.EMPTY_ARRAY, null, null)),
+            response,
+            Collections.singleton(responses.get(0).getId()),
+            null
+        );
 
         final PlainActionFuture<GetApiKeyResponse> failureListener = new PlainActionFuture<>();
         // for any other API key id, it must deny access
@@ -1476,28 +1530,11 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
         final boolean isUpdated = nullRoleDescriptors == false || metadataChanged;
         assertEquals(isUpdated, response.isUpdated());
 
-        final PlainActionFuture<GetApiKeyResponse> getListener = new PlainActionFuture<>();
-        client().filterWithHeader(
-            Collections.singletonMap("Authorization", basicAuthHeaderValue(TEST_USER_NAME, TEST_PASSWORD_SECURE_STRING))
-        ).execute(GetApiKeyAction.INSTANCE, GetApiKeyRequest.usingApiKeyId(apiKeyId, false), getListener);
-        final GetApiKeyResponse getResponse = getListener.get();
-        assertEquals(1, getResponse.getApiKeyInfos().length);
-        // When metadata for the update request is null (i.e., absent), we don't overwrite old metadata with it
-        final var expectedMetadata = request.getMetadata() != null ? request.getMetadata() : createdApiKey.v2();
-        assertEquals(expectedMetadata == null ? Map.of() : expectedMetadata, getResponse.getApiKeyInfos()[0].getMetadata());
-        assertEquals(TEST_USER_NAME, getResponse.getApiKeyInfos()[0].getUsername());
-        assertEquals("file", getResponse.getApiKeyInfos()[0].getRealm());
-
         // Test authenticate works with updated API key
         final var authResponse = authenticateWithApiKey(apiKeyId, createdApiKey.v1().getKey());
         assertThat(authResponse.get(User.Fields.USERNAME.getPreferredName()), equalTo(TEST_USER_NAME));
 
         // Document updated as expected
-        final var updatedApiKeyDoc = getApiKeyDocument(apiKeyId);
-        expectMetadataForApiKey(expectedMetadata, updatedApiKeyDoc);
-        expectRoleDescriptorsForApiKey("limited_by_role_descriptors", expectedLimitedByRoleDescriptors, updatedApiKeyDoc);
-        final var expectedRoleDescriptors = nullRoleDescriptors ? List.of(DEFAULT_API_KEY_ROLE_DESCRIPTOR) : newRoleDescriptors;
-        expectRoleDescriptorsForApiKey("role_descriptors", expectedRoleDescriptors, updatedApiKeyDoc);
         final Map<String, Object> expectedCreator = new HashMap<>();
         expectedCreator.put("principal", TEST_USER_NAME);
         expectedCreator.put("full_name", null);
@@ -1505,7 +1542,22 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
         expectedCreator.put("metadata", Map.of());
         expectedCreator.put("realm_type", "file");
         expectedCreator.put("realm", "file");
-        expectCreatorForApiKey(expectedCreator, updatedApiKeyDoc);
+        final var expectedMetadata = request.getMetadata() != null ? request.getMetadata() : createdApiKey.v2();
+        final var expectedRoleDescriptors = nullRoleDescriptors ? List.of(DEFAULT_API_KEY_ROLE_DESCRIPTOR) : newRoleDescriptors;
+
+        expectAttributesForApiKey(
+            apiKeyId,
+            Map.of(
+                ApiKeyAttribute.CREATOR,
+                expectedCreator,
+                ApiKeyAttribute.METADATA,
+                expectedMetadata == null ? Map.of() : expectedMetadata,
+                ApiKeyAttribute.ASSIGNED_ROLE_DESCRIPTORS,
+                expectedRoleDescriptors,
+                ApiKeyAttribute.LIMITED_BY_ROLE_DESCRIPTORS,
+                expectedLimitedByRoleDescriptors
+            )
+        );
 
         // Check if update resulted in API key role going from `monitor` to `all` cluster privilege and assert that action that requires
         // `all` is authorized or denied accordingly
@@ -1556,10 +1608,17 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
             )
         );
         for (String apiKeyId : apiKeyIds) {
-            final Map<String, Object> doc = getApiKeyDocument(apiKeyId);
-            expectRoleDescriptorsForApiKey("role_descriptors", newRoleDescriptors, doc);
-            expectRoleDescriptorsForApiKey("limited_by_role_descriptors", expectedLimitedByRoleDescriptors, doc);
-            expectMetadataForApiKey(newMetadata, doc);
+            expectAttributesForApiKey(
+                apiKeyId,
+                Map.of(
+                    ApiKeyAttribute.METADATA,
+                    newMetadata,
+                    ApiKeyAttribute.ASSIGNED_ROLE_DESCRIPTORS,
+                    newRoleDescriptors,
+                    ApiKeyAttribute.LIMITED_BY_ROLE_DESCRIPTORS,
+                    expectedLimitedByRoleDescriptors
+                )
+            );
         }
 
         // Check that bulk update works when there are no actual updates
@@ -1581,10 +1640,17 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
         assertThat(response.getNoops(), containsInAnyOrder(apiKeyIds.toArray()));
         assertThat(response.getErrorDetails().keySet(), containsInAnyOrder(notFoundIds.toArray()));
         for (String apiKeyId : apiKeyIds) {
-            final Map<String, Object> doc = getApiKeyDocument(apiKeyId);
-            expectRoleDescriptorsForApiKey("role_descriptors", newRoleDescriptors, doc);
-            expectRoleDescriptorsForApiKey("limited_by_role_descriptors", expectedLimitedByRoleDescriptors, doc);
-            expectMetadataForApiKey(newMetadata, doc);
+            expectAttributesForApiKey(
+                apiKeyId,
+                Map.of(
+                    ApiKeyAttribute.METADATA,
+                    newMetadata,
+                    ApiKeyAttribute.ASSIGNED_ROLE_DESCRIPTORS,
+                    newRoleDescriptors,
+                    ApiKeyAttribute.LIMITED_BY_ROLE_DESCRIPTORS,
+                    expectedLimitedByRoleDescriptors
+                )
+            );
         }
 
         // Check that bulk update works when some or all updates result in errors
@@ -1670,11 +1736,11 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
             "all"
         );
         final List<String> firstGenerationApiKeyIds = firstGenerationApiKeys.v1().stream().map(CreateApiKeyResponse::getId).toList();
-        expectRoleDescriptorsForApiKeys(
-            "limited_by_role_descriptors",
-            Set.of(firstGenerationRoleDescriptor),
-            firstGenerationApiKeyIds.stream().map(this::getApiKeyDocument).toList()
+        expectAttributesForApiKeys(
+            firstGenerationApiKeyIds,
+            Map.of(ApiKeyAttribute.LIMITED_BY_ROLE_DESCRIPTORS, Set.of(firstGenerationRoleDescriptor))
         );
+
         // Update user's permissions and create new API keys for the user. The new API keys will have different limited-by role descriptors
         final List<String> secondGenerationClusterPrivileges = randomValueOtherThan(firstGenerationClusterPrivileges, () -> {
             final List<String> privs = new ArrayList<>(randomSubsetOf(ClusterPrivilegeResolver.names()));
@@ -1693,10 +1759,9 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
             "all"
         );
         final List<String> secondGenerationApiKeyIds = secondGenerationApiKeys.v1().stream().map(CreateApiKeyResponse::getId).toList();
-        expectRoleDescriptorsForApiKeys(
-            "limited_by_role_descriptors",
-            Set.of(secondGenerationRoleDescriptor),
-            secondGenerationApiKeyIds.stream().map(this::getApiKeyDocument).toList()
+        expectAttributesForApiKeys(
+            secondGenerationApiKeyIds,
+            Map.of(ApiKeyAttribute.LIMITED_BY_ROLE_DESCRIPTORS, Set.of(secondGenerationRoleDescriptor))
         );
         // Update user role then bulk update all API keys. This should result in new limited-by role descriptors for all API keys
         final List<String> allIds = Stream.concat(firstGenerationApiKeyIds.stream(), secondGenerationApiKeyIds.stream()).toList();
@@ -1722,11 +1787,7 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
         assertThat(response.getErrorDetails(), anEmptyMap());
         assertThat(response.getNoops(), empty());
         assertThat(response.getUpdated(), containsInAnyOrder(allIds.toArray()));
-        expectRoleDescriptorsForApiKeys(
-            "limited_by_role_descriptors",
-            Set.of(finalRoleDescriptor),
-            allIds.stream().map(this::getApiKeyDocument).toList()
-        );
+        expectAttributesForApiKeys(allIds, Map.of(ApiKeyAttribute.LIMITED_BY_ROLE_DESCRIPTORS, Set.of(finalRoleDescriptor)));
     }
 
     public void testUpdateApiKeysAutoUpdatesUserFields() throws Exception {
@@ -1755,7 +1816,7 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
             "all"
         ).v1().get(0);
         final String apiKeyId = createdApiKey.getId();
-        expectRoleDescriptorsForApiKey("limited_by_role_descriptors", Set.of(roleDescriptorBeforeUpdate), getApiKeyDocument(apiKeyId));
+        expectAttributesForApiKey(apiKeyId, Map.of(ApiKeyAttribute.LIMITED_BY_ROLE_DESCRIPTORS, Set.of(roleDescriptorBeforeUpdate)));
 
         final List<String> newClusterPrivileges = randomValueOtherThan(clusterPrivileges, () -> {
             final List<String> privs = new ArrayList<>(randomSubsetOf(ClusterPrivilegeResolver.names()));
@@ -1777,7 +1838,7 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
 
         assertNotNull(response);
         assertTrue(response.isUpdated());
-        expectRoleDescriptorsForApiKey("limited_by_role_descriptors", Set.of(roleDescriptorAfterUpdate), getApiKeyDocument(apiKeyId));
+        expectAttributesForApiKey(apiKeyId, Map.of(ApiKeyAttribute.LIMITED_BY_ROLE_DESCRIPTORS, Set.of(roleDescriptorAfterUpdate)));
 
         // Update user role name only
         final RoleDescriptor roleDescriptorWithNewName = putRoleWithClusterPrivileges(
@@ -1796,8 +1857,6 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
 
         assertNotNull(response);
         assertTrue(response.isUpdated());
-        final Map<String, Object> updatedApiKeyDoc = getApiKeyDocument(apiKeyId);
-        expectRoleDescriptorsForApiKey("limited_by_role_descriptors", Set.of(roleDescriptorWithNewName), updatedApiKeyDoc);
         final Map<String, Object> expectedCreator = new HashMap<>();
         expectedCreator.put("principal", updatedUser.principal());
         expectedCreator.put("full_name", updatedUser.fullName());
@@ -1805,7 +1864,10 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
         expectedCreator.put("metadata", updatedUser.metadata());
         expectedCreator.put("realm_type", "native");
         expectedCreator.put("realm", "index");
-        expectCreatorForApiKey(expectedCreator, updatedApiKeyDoc);
+        expectAttributesForApiKey(
+            apiKeyId,
+            Map.of(ApiKeyAttribute.CREATOR, expectedCreator, ApiKeyAttribute.LIMITED_BY_ROLE_DESCRIPTORS, Set.of(roleDescriptorWithNewName))
+        );
     }
 
     public void testUpdateApiKeysNotFoundScenarios() throws Exception {
@@ -2104,6 +2166,7 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
                 legacySuperuserRoleDescriptor
             )
         );
+        // raw document has the legacy superuser role descriptor
         expectRoleDescriptorsForApiKey("limited_by_role_descriptors", legacySuperuserRoleDescriptor, getApiKeyDocument(apiKeyId));
 
         final Set<RoleDescriptor> currentSuperuserRoleDescriptors = Set.of(ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR);
@@ -2117,7 +2180,7 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
                 currentSuperuserRoleDescriptors
             )
         );
-        expectRoleDescriptorsForApiKey("limited_by_role_descriptors", currentSuperuserRoleDescriptors, getApiKeyDocument(apiKeyId));
+        expectAttributesForApiKey(apiKeyId, Map.of(ApiKeyAttribute.LIMITED_BY_ROLE_DESCRIPTORS, currentSuperuserRoleDescriptors));
         // Second update is noop because role descriptors were auto-updated by the previous request
         assertSingleNoop(
             apiKeyId,
@@ -2226,6 +2289,56 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
         assertThat(ex.getMessage(), containsString("no API key owned by requesting user found for ID [" + request.getId() + "]"));
     }
 
+    private enum ApiKeyAttribute {
+        CREATOR,
+        METADATA,
+        ASSIGNED_ROLE_DESCRIPTORS,
+        LIMITED_BY_ROLE_DESCRIPTORS
+    }
+
+    // Check attributes with both the raw document and the get api key response whenever possible
+    @SuppressWarnings("unchecked")
+    private void expectAttributesForApiKey(String apiKeyId, Map<ApiKeyAttribute, Object> attributes) throws IOException {
+        final Map<String, Object> apiKeyDocMap = getApiKeyDocument(apiKeyId);
+        final PlainActionFuture<GetApiKeyResponse> future = new PlainActionFuture<>();
+        client().execute(GetApiKeyAction.INSTANCE, GetApiKeyRequest.usingApiKeyId(apiKeyId, false), future);
+        final GetApiKeyResponse getApiKeyResponse = future.actionGet();
+        assertThat(getApiKeyResponse.getApiKeyInfos(), arrayWithSize(1));
+        final ApiKey apiKeyInfo = getApiKeyResponse.getApiKeyInfos()[0];
+
+        for (Map.Entry<ApiKeyAttribute, Object> entry : attributes.entrySet()) {
+            switch (entry.getKey()) {
+                case CREATOR -> {
+                    final var creatorMap = (Map<String, Object>) entry.getValue();
+                    expectCreatorForApiKey(creatorMap, apiKeyDocMap);
+                    assertThat(creatorMap.get("principal"), equalTo(apiKeyInfo.getUsername()));
+                    assertThat(creatorMap.get("realm"), equalTo(apiKeyInfo.getRealm()));
+                }
+                case METADATA -> {
+                    final var metadata = (Map<String, Object>) entry.getValue();
+                    expectMetadataForApiKey(metadata, apiKeyDocMap);
+                    assertThat(metadata, equalTo(apiKeyInfo.getMetadata()));
+                }
+                case ASSIGNED_ROLE_DESCRIPTORS -> {
+                    final var expectedRoleDescriptors = (Collection<RoleDescriptor>) entry.getValue();
+                    expectRoleDescriptorsForApiKey("role_descriptors", expectedRoleDescriptors, apiKeyDocMap);
+                    assertThat(expectedRoleDescriptors, containsInAnyOrder(apiKeyInfo.getRoleDescriptors().toArray(RoleDescriptor[]::new)));
+                }
+                case LIMITED_BY_ROLE_DESCRIPTORS -> {
+                    final var expectedRoleDescriptors = (Collection<RoleDescriptor>) entry.getValue();
+                    expectRoleDescriptorsForApiKey("limited_by_role_descriptors", expectedRoleDescriptors, apiKeyDocMap);
+                }
+                default -> throw new IllegalStateException("unexpected attribute name");
+            }
+        }
+    }
+
+    private void expectAttributesForApiKeys(List<String> apiKeyIds, Map<ApiKeyAttribute, Object> attributes) throws IOException {
+        for (String apiKeyId : apiKeyIds) {
+            expectAttributesForApiKey(apiKeyId, attributes);
+        }
+    }
+
     private void expectMetadataForApiKey(final Map<String, Object> expectedMetadata, final Map<String, Object> actualRawApiKeyDoc) {
         assertNotNull(actualRawApiKeyDoc);
         @SuppressWarnings("unchecked")
@@ -2263,16 +2376,6 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
         }
     }
 
-    private void expectRoleDescriptorsForApiKeys(
-        final String roleDescriptorType,
-        final Collection<RoleDescriptor> expectedRoleDescriptors,
-        final List<Map<String, Object>> actualRawApiKeyDocs
-    ) throws IOException {
-        for (Map<String, Object> actualDoc : actualRawApiKeyDocs) {
-            expectRoleDescriptorsForApiKey(roleDescriptorType, expectedRoleDescriptors, actualDoc);
-        }
-    }
-
     private Map<String, Object> getApiKeyDocument(String apiKeyId) {
         return client().execute(GetAction.INSTANCE, new GetRequest(SECURITY_MAIN_ALIAS, apiKeyId)).actionGet().getSource();
     }
@@ -2333,11 +2436,21 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
         int expectedNumberOfApiKeys,
         List<CreateApiKeyResponse> responses,
         List<Map<String, Object>> metadatas,
+        List<RoleDescriptor> expectedRoleDescriptors,
         GetApiKeyResponse response,
         Set<String> validApiKeyIds,
         List<String> invalidatedApiKeyIds
     ) {
-        verifyGetResponse(ES_TEST_ROOT_USER, expectedNumberOfApiKeys, responses, metadatas, response, validApiKeyIds, invalidatedApiKeyIds);
+        verifyGetResponse(
+            ES_TEST_ROOT_USER,
+            expectedNumberOfApiKeys,
+            responses,
+            metadatas,
+            expectedRoleDescriptors,
+            response,
+            validApiKeyIds,
+            invalidatedApiKeyIds
+        );
     }
 
     private void verifyGetResponse(
@@ -2345,6 +2458,7 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
         int expectedNumberOfApiKeys,
         List<CreateApiKeyResponse> responses,
         List<Map<String, Object>> metadatas,
+        List<RoleDescriptor> expectedRoleDescriptors,
         GetApiKeyResponse response,
         Set<String> validApiKeyIds,
         List<String> invalidatedApiKeyIds
@@ -2354,6 +2468,7 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
             expectedNumberOfApiKeys,
             responses,
             metadatas,
+            expectedRoleDescriptors,
             response,
             validApiKeyIds,
             invalidatedApiKeyIds
@@ -2365,6 +2480,7 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
         int expectedNumberOfApiKeys,
         List<CreateApiKeyResponse> responses,
         List<Map<String, Object>> metadatas,
+        List<RoleDescriptor> expectedRoleDescriptors,
         GetApiKeyResponse response,
         Set<String> validApiKeyIds,
         List<String> invalidatedApiKeyIds
@@ -2413,6 +2529,13 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
                 assertThat(apiKey.getMetadata(), equalTo(metadata == null ? Map.of() : metadata));
             }
         }
+        Arrays.stream(response.getApiKeyInfos())
+            .forEach(
+                apiKeyInfo -> assertThat(
+                    apiKeyInfo.getRoleDescriptors(),
+                    containsInAnyOrder(expectedRoleDescriptors.toArray(RoleDescriptor[]::new))
+                )
+            );
     }
 
     private Tuple<CreateApiKeyResponse, Map<String, Object>> createApiKey(String user, TimeValue expiration) {

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java

@@ -840,7 +840,7 @@ public class ApiKeyService {
             if (apiKeyAuthCache != null) {
                 apiKeyAuthCache.invalidate(docId);
             }
-            listener.onResponse(AuthenticationResult.unsuccessful("api key has been invalidated", null));
+            listener.onResponse(AuthenticationResult.unsuccessful("api key [" + credentials.getId() + "] has been invalidated", null));
         } else {
             if (apiKeyDoc.hash == null) {
                 throw new IllegalStateException("api key hash is missing");
@@ -1212,7 +1212,7 @@ public class ApiKeyService {
                 apiKeyIds,
                 true,
                 false,
-                ApiKeyService::convertSearchHitToApiKeyInfo,
+                this::convertSearchHitToApiKeyInfo,
                 ActionListener.wrap(apiKeys -> {
                     if (apiKeys.isEmpty()) {
                         logger.debug(
@@ -1593,7 +1593,7 @@ public class ApiKeyService {
             apiKeyIds,
             false,
             false,
-            ApiKeyService::convertSearchHitToApiKeyInfo,
+            this::convertSearchHitToApiKeyInfo,
             ActionListener.wrap(apiKeyInfos -> {
                 if (apiKeyInfos.isEmpty()) {
                     logger.debug(
@@ -1636,7 +1636,7 @@ public class ApiKeyService {
                             return;
                         }
                         final List<QueryApiKeyResponse.Item> apiKeyItem = Arrays.stream(searchResponse.getHits().getHits())
-                            .map(ApiKeyService::convertSearchHitToQueryItem)
+                            .map(this::convertSearchHitToQueryItem)
                             .toList();
                         listener.onResponse(new QueryApiKeyResponse(total, apiKeyItem));
                     }, listener::onFailure)
@@ -1645,33 +1645,33 @@ public class ApiKeyService {
         }
     }
 
-    private static QueryApiKeyResponse.Item convertSearchHitToQueryItem(SearchHit hit) {
+    private QueryApiKeyResponse.Item convertSearchHitToQueryItem(SearchHit hit) {
         return new QueryApiKeyResponse.Item(convertSearchHitToApiKeyInfo(hit), hit.getSortValues());
     }
 
-    private static ApiKey convertSearchHitToApiKeyInfo(SearchHit hit) {
-        Map<String, Object> source = hit.getSourceAsMap();
-        String name = (String) source.get("name");
-        String id = hit.getId();
-        Long creation = (Long) source.get("creation_time");
-        Long expiration = (Long) source.get("expiration_time");
-        Boolean invalidated = (Boolean) source.get("api_key_invalidated");
-        @SuppressWarnings("unchecked")
-        String username = (String) ((Map<String, Object>) source.get("creator")).get("principal");
-        @SuppressWarnings("unchecked")
-        String realm = (String) ((Map<String, Object>) source.get("creator")).get("realm");
-        @SuppressWarnings("unchecked")
-        Map<String, Object> metadata = (Map<String, Object>) source.get("metadata_flattened");
+    private ApiKey convertSearchHitToApiKeyInfo(SearchHit hit) {
+        final ApiKeyDoc apiKeyDoc = convertSearchHitToVersionedApiKeyDoc(hit).doc;
+        final String apiKeyId = hit.getId();
+        final Map<String, Object> metadata = apiKeyDoc.metadataFlattened != null
+            ? XContentHelper.convertToMap(apiKeyDoc.metadataFlattened, false, XContentType.JSON).v2()
+            : Map.of();
+
+        final List<RoleDescriptor> roleDescriptors = parseRoleDescriptorsBytes(
+            apiKeyId,
+            apiKeyDoc.roleDescriptorsBytes,
+            RoleReference.ApiKeyRoleType.ASSIGNED
+        );
 
         return new ApiKey(
-            name,
-            id,
-            Instant.ofEpochMilli(creation),
-            (expiration != null) ? Instant.ofEpochMilli(expiration) : null,
-            invalidated,
-            username,
-            realm,
-            metadata
+            apiKeyDoc.name,
+            apiKeyId,
+            Instant.ofEpochMilli(apiKeyDoc.creationTime),
+            apiKeyDoc.expirationTime != -1 ? Instant.ofEpochMilli(apiKeyDoc.expirationTime) : null,
+            apiKeyDoc.invalidated,
+            (String) apiKeyDoc.creator.get("principal"),
+            (String) apiKeyDoc.creator.get("realm"),
+            metadata,
+            roleDescriptors
         );
     }
 

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java

@@ -82,13 +82,13 @@ import org.elasticsearch.xpack.core.security.authc.RealmDomain;
 import org.elasticsearch.xpack.core.security.authc.support.AuthenticationContextSerializer;
 import org.elasticsearch.xpack.core.security.authc.support.Hasher;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptorTests;
 import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege;
 import org.elasticsearch.xpack.core.security.authz.store.RoleReference;
 import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.security.authc.ApiKeyService.ApiKeyCredentials;
 import org.elasticsearch.xpack.security.authc.ApiKeyService.ApiKeyDoc;
 import org.elasticsearch.xpack.security.authc.ApiKeyService.CachedApiKeyHashResult;
-import org.elasticsearch.xpack.security.authz.RoleDescriptorTests;
 import org.elasticsearch.xpack.security.authz.store.NativePrivilegeStore;
 import org.elasticsearch.xpack.security.support.CacheInvalidatorRegistry;
 import org.elasticsearch.xpack.security.support.FeatureNotEnabledException;

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestGetApiKeyActionTests.java

@@ -32,6 +32,7 @@ import org.elasticsearch.xpack.core.security.action.apikey.ApiKey;
 import org.elasticsearch.xpack.core.security.action.apikey.ApiKeyTests;
 import org.elasticsearch.xpack.core.security.action.apikey.GetApiKeyRequest;
 import org.elasticsearch.xpack.core.security.action.apikey.GetApiKeyResponse;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
@@ -40,6 +41,7 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.xpack.core.security.authz.RoleDescriptorTests.randomUniquelyNamedRoleDescriptors;
 import static org.hamcrest.Matchers.arrayContaining;
 import static org.hamcrest.Matchers.is;
 import static org.mockito.Mockito.mock;
@@ -87,9 +89,10 @@ public class RestGetApiKeyActionTests extends ESTestCase {
         final Instant expiration = randomFrom(Arrays.asList(null, Instant.now().plus(10, ChronoUnit.DAYS)));
         @SuppressWarnings("unchecked")
         final Map<String, Object> metadata = ApiKeyTests.randomMetadata();
+        final List<RoleDescriptor> roleDescriptors = randomUniquelyNamedRoleDescriptors(0, 3);
         final GetApiKeyResponse getApiKeyResponseExpected = new GetApiKeyResponse(
             Collections.singletonList(
-                new ApiKey("api-key-name-1", "api-key-id-1", creation, expiration, false, "user-x", "realm-1", metadata)
+                new ApiKey("api-key-name-1", "api-key-id-1", creation, expiration, false, "user-x", "realm-1", metadata, roleDescriptors)
             )
         );
 
@@ -140,7 +143,17 @@ public class RestGetApiKeyActionTests extends ESTestCase {
                 assertThat(
                     actual.getApiKeyInfos(),
                     arrayContaining(
-                        new ApiKey("api-key-name-1", "api-key-id-1", creation, expiration, false, "user-x", "realm-1", metadata)
+                        new ApiKey(
+                            "api-key-name-1",
+                            "api-key-id-1",
+                            creation,
+                            expiration,
+                            false,
+                            "user-x",
+                            "realm-1",
+                            metadata,
+                            roleDescriptors
+                        )
                     )
                 );
             }
@@ -177,7 +190,8 @@ public class RestGetApiKeyActionTests extends ESTestCase {
             false,
             "user-x",
             "realm-1",
-            ApiKeyTests.randomMetadata()
+            ApiKeyTests.randomMetadata(),
+            randomUniquelyNamedRoleDescriptors(0, 3)
         );
         final ApiKey apiKey2 = new ApiKey(
             "api-key-name-2",
@@ -187,7 +201,8 @@ public class RestGetApiKeyActionTests extends ESTestCase {
             false,
             "user-y",
             "realm-1",
-            ApiKeyTests.randomMetadata()
+            ApiKeyTests.randomMetadata(),
+            randomUniquelyNamedRoleDescriptors(0, 3)
         );
         final GetApiKeyResponse getApiKeyResponseExpectedWhenOwnerFlagIsTrue = new GetApiKeyResponse(Collections.singletonList(apiKey1));
         final GetApiKeyResponse getApiKeyResponseExpectedWhenOwnerFlagIsFalse = new GetApiKeyResponse(List.of(apiKey1, apiKey2));