Add support for the `type` parameter to the Query API Key API (#103695)

This adds support for the type parameter to the Query API key API.
The type for an API Key can currently be either rest or cross_cluster.

Relates: #101691

docs/reference/rest-api/security/query-api-key.asciidoc

@@ -64,6 +64,11 @@ You can query the following public values associated with an API key.
 `id`::
 ID of the API key. Note `id` must be queried with the <<query-dsl-ids-query,`ids`>> query.
 
+`type`::
+API keys can be of type `rest`, if created via the <<security-api-create-api-key, Create API key>> or
+the <<security-api-grant-api-key, Grant API key>> APIs, or of type `cross_cluster` if created via
+the <<security-api-create-cross-cluster-api-key, Create Cross-Cluster API key>> API.
+
 `name`::
 Name of the API key.
 

x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryApiKeyIT.java

@@ -9,8 +9,10 @@ package org.elasticsearch.xpack.security;
 
 import org.apache.http.HttpHeaders;
 import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.test.XContentTestUtils;
@@ -21,6 +23,7 @@ import java.nio.charset.StandardCharsets;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Base64;
+import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -43,6 +46,8 @@ public class QueryApiKeyIT extends SecurityInBasicRestTestCase {
     private static final String API_KEY_ADMIN_AUTH_HEADER = "Basic YXBpX2tleV9hZG1pbjpzZWN1cml0eS10ZXN0LXBhc3N3b3Jk";
     private static final String API_KEY_USER_AUTH_HEADER = "Basic YXBpX2tleV91c2VyOnNlY3VyaXR5LXRlc3QtcGFzc3dvcmQ=";
     private static final String TEST_USER_AUTH_HEADER = "Basic c2VjdXJpdHlfdGVzdF91c2VyOnNlY3VyaXR5LXRlc3QtcGFzc3dvcmQ=";
+    private static final String SYSTEM_WRITE_ROLE_NAME = "system_write";
+    private static final String SUPERUSER_WITH_SYSTEM_WRITE = "superuser_with_system_write";
 
     public void testQuery() throws IOException {
         createApiKeys();
@@ -297,6 +302,71 @@ public class QueryApiKeyIT extends SecurityInBasicRestTestCase {
         assertThat(responseMap2.get("count"), equalTo(0));
     }
 
+    public void testTypeField() throws Exception {
+        final List<String> allApiKeyIds = new ArrayList<>(7);
+        for (int i = 0; i < 7; i++) {
+            allApiKeyIds.add(
+                createApiKey("typed_key_" + i, Map.of(), randomFrom(API_KEY_ADMIN_AUTH_HEADER, API_KEY_USER_AUTH_HEADER)).v1()
+            );
+        }
+        List<String> apiKeyIdsSubset = randomSubsetOf(allApiKeyIds);
+        List<String> apiKeyIdsSubsetDifference = new ArrayList<>(allApiKeyIds);
+        apiKeyIdsSubsetDifference.removeAll(apiKeyIdsSubset);
+
+        List<String> apiKeyRestTypeQueries = List.of("""
+            {"query": {"term": {"type": "rest" }}}""", """
+            {"query": {"bool": {"must_not": [{"term": {"type": "cross_cluster"}}, {"term": {"type": "other"}}]}}}""", """
+            {"query": {"prefix": {"type": "re" }}}""", """
+            {"query": {"wildcard": {"type": "r*t" }}}""", """
+            {"query": {"range": {"type": {"gte": "raaa", "lte": "rzzz"}}}}""");
+
+        for (String query : apiKeyRestTypeQueries) {
+            assertQuery(API_KEY_ADMIN_AUTH_HEADER, query, apiKeys -> {
+                assertThat(
+                    apiKeys.stream().map(k -> (String) k.get("id")).toList(),
+                    containsInAnyOrder(allApiKeyIds.toArray(new String[0]))
+                );
+            });
+        }
+
+        createSystemWriteRole(SYSTEM_WRITE_ROLE_NAME);
+        String systemWriteCreds = createUser(SUPERUSER_WITH_SYSTEM_WRITE, new String[] { "superuser", SYSTEM_WRITE_ROLE_NAME });
+
+        // test keys with no "type" field are still considered of type "rest"
+        // this is so in order to accommodate pre-8.9 API keys which where all of type "rest" implicitly
+        updateApiKeys(systemWriteCreds, "ctx._source.remove('type');", apiKeyIdsSubset);
+        for (String query : apiKeyRestTypeQueries) {
+            assertQuery(API_KEY_ADMIN_AUTH_HEADER, query, apiKeys -> {
+                assertThat(
+                    apiKeys.stream().map(k -> (String) k.get("id")).toList(),
+                    containsInAnyOrder(allApiKeyIds.toArray(new String[0]))
+                );
+            });
+        }
+
+        // but the same keys with type "other" are NOT of type "rest"
+        updateApiKeys(systemWriteCreds, "ctx._source['type']='other';", apiKeyIdsSubset);
+        for (String query : apiKeyRestTypeQueries) {
+            assertQuery(API_KEY_ADMIN_AUTH_HEADER, query, apiKeys -> {
+                assertThat(
+                    apiKeys.stream().map(k -> (String) k.get("id")).toList(),
+                    containsInAnyOrder(apiKeyIdsSubsetDifference.toArray(new String[0]))
+                );
+            });
+        }
+        // the complement set is not of type "rest" if it is "cross_cluster"
+        updateApiKeys(systemWriteCreds, "ctx._source['type']='rest';", apiKeyIdsSubset);
+        updateApiKeys(systemWriteCreds, "ctx._source['type']='cross_cluster';", apiKeyIdsSubsetDifference);
+        for (String query : apiKeyRestTypeQueries) {
+            assertQuery(API_KEY_ADMIN_AUTH_HEADER, query, apiKeys -> {
+                assertThat(
+                    apiKeys.stream().map(k -> (String) k.get("id")).toList(),
+                    containsInAnyOrder(apiKeyIdsSubset.toArray(new String[0]))
+                );
+            });
+        }
+    }
+
     @SuppressWarnings("unchecked")
     public void testSort() throws IOException {
         final String authHeader = randomFrom(API_KEY_ADMIN_AUTH_HEADER, API_KEY_USER_AUTH_HEADER);
@@ -598,10 +668,73 @@ public class QueryApiKeyIT extends SecurityInBasicRestTestCase {
         return tuple.v1();
     }
 
-    private void createUser(String name) throws IOException {
-        final Request request = new Request("POST", "/_security/user/" + name);
-        request.setJsonEntity("""
-            {"password":"super-strong-password","roles":[]}""");
-        assertOK(adminClient().performRequest(request));
+    private String createUser(String username) throws IOException {
+        return createUser(username, new String[0]);
+    }
+
+    private String createUser(String username, String[] roles) throws IOException {
+        final Request request = new Request("POST", "/_security/user/" + username);
+        Map<String, Object> body = Map.ofEntries(Map.entry("roles", roles), Map.entry("password", "super-strong-password".toString()));
+        request.setJsonEntity(XContentTestUtils.convertToXContent(body, XContentType.JSON).utf8ToString());
+        Response response = adminClient().performRequest(request);
+        assertOK(response);
+        return basicAuthHeaderValue(username, new SecureString("super-strong-password".toCharArray()));
+    }
+
+    private void createSystemWriteRole(String roleName) throws IOException {
+        final Request addRole = new Request("POST", "/_security/role/" + roleName);
+        addRole.setJsonEntity("""
+            {
+              "indices": [
+                {
+                  "names": [ "*" ],
+                  "privileges": ["all"],
+                  "allow_restricted_indices" : true
+                }
+              ]
+            }""");
+        Response response = adminClient().performRequest(addRole);
+        assertOK(response);
+    }
+
+    private void expectWarnings(Request request, String... expectedWarnings) {
+        final Set<String> expected = Set.of(expectedWarnings);
+        RequestOptions options = request.getOptions().toBuilder().setWarningsHandler(warnings -> {
+            final Set<String> actual = Set.copyOf(warnings);
+            // Return true if the warnings aren't what we expected; the client will treat them as a fatal error.
+            return actual.equals(expected) == false;
+        }).build();
+        request.setOptions(options);
+    }
+
+    private void updateApiKeys(String creds, String script, Collection<String> ids) throws IOException {
+        if (ids.isEmpty()) {
+            return;
+        }
+        final Request request = new Request("POST", "/.security/_update_by_query?refresh=true&wait_for_completion=true");
+        request.setJsonEntity(Strings.format("""
+            {
+              "script": {
+                "source": "%s",
+                "lang": "painless"
+              },
+              "query": {
+                "bool": {
+                  "must": [
+                    {"term": {"doc_type": "api_key"}},
+                    {"ids": {"values": %s}}
+                  ]
+                }
+              }
+            }
+            """, script, ids.stream().map(id -> "\"" + id + "\"").collect(Collectors.toList())));
+        request.setOptions(request.getOptions().toBuilder().addHeader(HttpHeaders.AUTHORIZATION, creds));
+        expectWarnings(
+            request,
+            "this request accesses system indices: [.security-7],"
+                + " but in a future major version, direct access to system indices will be prevented by default"
+        );
+        Response response = client().performRequest(request);
+        assertOK(response);
     }
 }

x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java

@@ -35,8 +35,10 @@ import org.junit.Before;
 import java.io.IOException;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -54,9 +56,11 @@ import static org.hamcrest.Matchers.emptyString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasEntry;
+import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.iterableWithSize;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
@@ -703,6 +707,73 @@ public class ApiKeyRestIT extends SecurityOnTrialLicenseRestTestCase {
 
     }
 
+    @SuppressWarnings("unchecked")
+    public void testQueryCrossClusterApiKeysByType() throws IOException {
+        final List<String> apiKeyIds = new ArrayList<>(3);
+        for (int i = 0; i < randomIntBetween(3, 5); i++) {
+            Request createRequest = new Request("POST", "/_security/cross_cluster/api_key");
+            createRequest.setJsonEntity(Strings.format("""
+                {
+                  "name": "test-cross-key-query-%d",
+                  "access": {
+                    "search": [
+                      {
+                        "names": [ "whatever" ]
+                      }
+                    ]
+                  },
+                  "metadata": { "tag": %d, "label": "rest" }
+                }""", i, i));
+            setUserForRequest(createRequest, MANAGE_SECURITY_USER, END_USER_PASSWORD);
+            ObjectPath createResponse = assertOKAndCreateObjectPath(client().performRequest(createRequest));
+            apiKeyIds.add(createResponse.evaluate("id"));
+        }
+        // the "cross_cluster" keys are not "rest" type
+        for (String restTypeQuery : List.of("""
+            {"query": {"term": {"type": "rest" }}}""", """
+            {"query": {"bool": {"must_not": {"term": {"type": "cross_cluster"}}}}}""", """
+            {"query": {"prefix": {"type": "re" }}}""", """
+            {"query": {"wildcard": {"type": "r*t" }}}""", """
+            {"query": {"range": {"type": {"gte": "raaa", "lte": "rzzz"}}}}""")) {
+            Request queryRequest = new Request("GET", "/_security/_query/api_key");
+            queryRequest.addParameter("with_limited_by", String.valueOf(randomBoolean()));
+            queryRequest.setJsonEntity(restTypeQuery);
+            setUserForRequest(queryRequest, MANAGE_API_KEY_USER, END_USER_PASSWORD);
+            ObjectPath queryResponse = assertOKAndCreateObjectPath(client().performRequest(queryRequest));
+            assertThat(queryResponse.evaluate("total"), is(0));
+            assertThat(queryResponse.evaluate("count"), is(0));
+            assertThat(queryResponse.evaluate("api_keys"), iterableWithSize(0));
+        }
+        for (String crossClusterTypeQuery : List.of("""
+            {"query": {"term": {"type": "cross_cluster" }}}""", """
+            {"query": {"bool": {"must_not": {"term": {"type": "rest"}}}}}""", """
+            {"query": {"prefix": {"type": "cro" }}}""", """
+            {"query": {"wildcard": {"type": "*oss_*er" }}}""", """
+            {"query": {"range": {"type": {"gte": "cross", "lte": "zzzz"}}}}""")) {
+            Request queryRequest = new Request("GET", "/_security/_query/api_key");
+            queryRequest.addParameter("with_limited_by", String.valueOf(randomBoolean()));
+            queryRequest.setJsonEntity(crossClusterTypeQuery);
+            setUserForRequest(queryRequest, MANAGE_API_KEY_USER, END_USER_PASSWORD);
+            ObjectPath queryResponse = assertOKAndCreateObjectPath(client().performRequest(queryRequest));
+            assertThat(queryResponse.evaluate("total"), is(apiKeyIds.size()));
+            assertThat(queryResponse.evaluate("count"), is(apiKeyIds.size()));
+            assertThat(queryResponse.evaluate("api_keys"), iterableWithSize(apiKeyIds.size()));
+            Iterator<?> apiKeys = ((List<?>) queryResponse.evaluate("api_keys")).iterator();
+            while (apiKeys.hasNext()) {
+                assertThat(apiKeyIds, hasItem((String) ((Map<String, Object>) apiKeys.next()).get("id")));
+            }
+        }
+        final Request queryRequest = new Request("GET", "/_security/_query/api_key");
+        queryRequest.addParameter("with_limited_by", String.valueOf(randomBoolean()));
+        queryRequest.setJsonEntity("""
+            {"query": {"bool": {"must": [{"term": {"type": "cross_cluster" }}, {"term": {"metadata.tag": 2}}]}}}""");
+        setUserForRequest(queryRequest, MANAGE_API_KEY_USER, END_USER_PASSWORD);
+        final ObjectPath queryResponse = assertOKAndCreateObjectPath(client().performRequest(queryRequest));
+        assertThat(queryResponse.evaluate("total"), is(1));
+        assertThat(queryResponse.evaluate("count"), is(1));
+        assertThat(queryResponse.evaluate("api_keys.0.name"), is("test-cross-key-query-2"));
+    }
+
     public void testCreateCrossClusterApiKey() throws IOException {
         final Request createRequest = new Request("POST", "/_security/cross_cluster/api_key");
         createRequest.setJsonEntity("""

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportQueryApiKeyAction.java

@@ -27,11 +27,26 @@ import org.elasticsearch.xpack.security.support.ApiKeyBoolQueryBuilder;
 import org.elasticsearch.xpack.security.support.ApiKeyFieldNameTranslators;
 
 import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_MAIN_ALIAS;
 
 public final class TransportQueryApiKeyAction extends HandledTransportAction<QueryApiKeyRequest, QueryApiKeyResponse> {
 
+    // API keys with no "type" field are implicitly of type "rest" (this is the case for all API Keys created before v8.9).
+    // The below runtime field ensures that the "type" field can be used by the {@link RestQueryApiKeyAction},
+    // while making the implicit "rest" type feature transparent to the caller (hence all keys are either "rest"
+    // or "cross_cluster", and the "type" is always set).
+    // This can be improved, to get rid of the runtime performance impact of the runtime field, by reindexing
+    // the api key docs and setting the "type" to "rest" if empty. But the infrastructure to run such a maintenance
+    // task on a system index (once the cluster version permits) is not currently available.
+    public static final String API_KEY_TYPE_RUNTIME_MAPPING_FIELD = "runtime_key_type";
+    private static final Map<String, Object> API_KEY_TYPE_RUNTIME_MAPPING = Map.of(
+        API_KEY_TYPE_RUNTIME_MAPPING_FIELD,
+        Map.of("type", "keyword", "script", Map.of("source", "emit(field('type').get(\"rest\"));"))
+    );
+
     private final ApiKeyService apiKeyService;
     private final SecurityContext securityContext;
 
@@ -66,12 +81,19 @@ public final class TransportQueryApiKeyAction extends HandledTransportAction<Que
             searchSourceBuilder.size(request.getSize());
         }
 
-        final ApiKeyBoolQueryBuilder apiKeyBoolQueryBuilder = ApiKeyBoolQueryBuilder.build(
-            request.getQueryBuilder(),
-            request.isFilterForCurrentUser() ? authentication : null
-        );
+        final AtomicBoolean accessesApiKeyTypeField = new AtomicBoolean(false);
+        final ApiKeyBoolQueryBuilder apiKeyBoolQueryBuilder = ApiKeyBoolQueryBuilder.build(request.getQueryBuilder(), fieldName -> {
+            if (API_KEY_TYPE_RUNTIME_MAPPING_FIELD.equals(fieldName)) {
+                accessesApiKeyTypeField.set(true);
+            }
+        }, request.isFilterForCurrentUser() ? authentication : null);
         searchSourceBuilder.query(apiKeyBoolQueryBuilder);
 
+        // only add the query-level runtime field to the search request if it's actually referring the "type" field
+        if (accessesApiKeyTypeField.get()) {
+            searchSourceBuilder.runtimeMappings(API_KEY_TYPE_RUNTIME_MAPPING);
+        }
+
         if (request.getFieldSortBuilders() != null) {
             translateFieldSortBuilders(request.getFieldSortBuilders(), searchSourceBuilder);
         }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/ApiKeyBoolQueryBuilder.java

@@ -28,6 +28,9 @@ import org.elasticsearch.xpack.security.authc.ApiKeyService;
 
 import java.io.IOException;
 import java.util.Set;
+import java.util.function.Consumer;
+
+import static org.elasticsearch.xpack.security.action.apikey.TransportQueryApiKeyAction.API_KEY_TYPE_RUNTIME_MAPPING_FIELD;
 
 public class ApiKeyBoolQueryBuilder extends BoolQueryBuilder {
 
@@ -36,10 +39,14 @@ public class ApiKeyBoolQueryBuilder extends BoolQueryBuilder {
         "_id",
         "doc_type",
         "name",
+        "type",
+        API_KEY_TYPE_RUNTIME_MAPPING_FIELD,
         "api_key_invalidated",
         "invalidation_time",
         "creation_time",
-        "expiration_time"
+        "expiration_time",
+        "creator.principal",
+        "creator.realm"
     );
 
     private ApiKeyBoolQueryBuilder() {}
@@ -56,17 +63,23 @@ public class ApiKeyBoolQueryBuilder extends BoolQueryBuilder {
      *
      * @param queryBuilder This represents the query parsed directly from the user input. It is validated
      *                     and transformed (see above).
+     * @param fieldNameVisitor This {@code Consumer} is invoked with all the (index-level) field names referred to in the passed-in query.
      * @param authentication The user's authentication object. If present, it will be used to filter the results
      *                       to only include API keys owned by the user.
      * @return A specialised query builder for API keys that is safe to run on the security index.
      */
-    public static ApiKeyBoolQueryBuilder build(QueryBuilder queryBuilder, @Nullable Authentication authentication) {
+    public static ApiKeyBoolQueryBuilder build(
+        QueryBuilder queryBuilder,
+        Consumer<String> fieldNameVisitor,
+        @Nullable Authentication authentication
+    ) {
         final ApiKeyBoolQueryBuilder finalQuery = new ApiKeyBoolQueryBuilder();
         if (queryBuilder != null) {
-            QueryBuilder processedQuery = doProcess(queryBuilder);
+            QueryBuilder processedQuery = doProcess(queryBuilder, fieldNameVisitor);
             finalQuery.must(processedQuery);
         }
         finalQuery.filter(QueryBuilders.termQuery("doc_type", "api_key"));
+        fieldNameVisitor.accept("doc_type");
 
         if (authentication != null) {
             if (authentication.isApiKey()) {
@@ -77,8 +90,10 @@ public class ApiKeyBoolQueryBuilder extends BoolQueryBuilder {
                 finalQuery.filter(QueryBuilders.idsQuery().addIds(apiKeyId));
             } else {
                 finalQuery.filter(QueryBuilders.termQuery("creator.principal", authentication.getEffectiveSubject().getUser().principal()));
+                fieldNameVisitor.accept("creator.principal");
                 final String[] realms = ApiKeyService.getOwnersRealmNames(authentication);
                 final QueryBuilder realmsQuery = ApiKeyService.filterForRealmNames(realms);
+                fieldNameVisitor.accept("creator.realm");
                 assert realmsQuery != null;
                 finalQuery.filter(realmsQuery);
             }
@@ -86,15 +101,15 @@ public class ApiKeyBoolQueryBuilder extends BoolQueryBuilder {
         return finalQuery;
     }
 
-    private static QueryBuilder doProcess(QueryBuilder qb) {
+    private static QueryBuilder doProcess(QueryBuilder qb, Consumer<String> fieldNameVisitor) {
         if (qb instanceof final BoolQueryBuilder query) {
             final BoolQueryBuilder newQuery = QueryBuilders.boolQuery()
                 .minimumShouldMatch(query.minimumShouldMatch())
                 .adjustPureNegative(query.adjustPureNegative());
-            query.must().stream().map(ApiKeyBoolQueryBuilder::doProcess).forEach(newQuery::must);
-            query.should().stream().map(ApiKeyBoolQueryBuilder::doProcess).forEach(newQuery::should);
-            query.mustNot().stream().map(ApiKeyBoolQueryBuilder::doProcess).forEach(newQuery::mustNot);
-            query.filter().stream().map(ApiKeyBoolQueryBuilder::doProcess).forEach(newQuery::filter);
+            query.must().stream().map(q -> ApiKeyBoolQueryBuilder.doProcess(q, fieldNameVisitor)).forEach(newQuery::must);
+            query.should().stream().map(q -> ApiKeyBoolQueryBuilder.doProcess(q, fieldNameVisitor)).forEach(newQuery::should);
+            query.mustNot().stream().map(q -> ApiKeyBoolQueryBuilder.doProcess(q, fieldNameVisitor)).forEach(newQuery::mustNot);
+            query.filter().stream().map(q -> ApiKeyBoolQueryBuilder.doProcess(q, fieldNameVisitor)).forEach(newQuery::filter);
             return newQuery;
         } else if (qb instanceof MatchAllQueryBuilder) {
             return qb;
@@ -102,29 +117,35 @@ public class ApiKeyBoolQueryBuilder extends BoolQueryBuilder {
             return qb;
         } else if (qb instanceof final TermQueryBuilder query) {
             final String translatedFieldName = ApiKeyFieldNameTranslators.translate(query.fieldName());
+            fieldNameVisitor.accept(translatedFieldName);
             return QueryBuilders.termQuery(translatedFieldName, query.value()).caseInsensitive(query.caseInsensitive());
         } else if (qb instanceof final ExistsQueryBuilder query) {
             final String translatedFieldName = ApiKeyFieldNameTranslators.translate(query.fieldName());
+            fieldNameVisitor.accept(translatedFieldName);
             return QueryBuilders.existsQuery(translatedFieldName);
         } else if (qb instanceof final TermsQueryBuilder query) {
             if (query.termsLookup() != null) {
                 throw new IllegalArgumentException("terms query with terms lookup is not supported for API Key query");
             }
             final String translatedFieldName = ApiKeyFieldNameTranslators.translate(query.fieldName());
+            fieldNameVisitor.accept(translatedFieldName);
             return QueryBuilders.termsQuery(translatedFieldName, query.getValues());
         } else if (qb instanceof final PrefixQueryBuilder query) {
             final String translatedFieldName = ApiKeyFieldNameTranslators.translate(query.fieldName());
+            fieldNameVisitor.accept(translatedFieldName);
             return QueryBuilders.prefixQuery(translatedFieldName, query.value()).caseInsensitive(query.caseInsensitive());
         } else if (qb instanceof final WildcardQueryBuilder query) {
             final String translatedFieldName = ApiKeyFieldNameTranslators.translate(query.fieldName());
+            fieldNameVisitor.accept(translatedFieldName);
             return QueryBuilders.wildcardQuery(translatedFieldName, query.value())
                 .caseInsensitive(query.caseInsensitive())
                 .rewrite(query.rewrite());
         } else if (qb instanceof final RangeQueryBuilder query) {
-            final String translatedFieldName = ApiKeyFieldNameTranslators.translate(query.fieldName());
             if (query.relation() != null) {
                 throw new IllegalArgumentException("range query with relation is not supported for API Key query");
             }
+            final String translatedFieldName = ApiKeyFieldNameTranslators.translate(query.fieldName());
+            fieldNameVisitor.accept(translatedFieldName);
             final RangeQueryBuilder newQuery = QueryBuilders.rangeQuery(translatedFieldName);
             if (query.format() != null) {
                 newQuery.format(query.format());
@@ -159,9 +180,7 @@ public class ApiKeyBoolQueryBuilder extends BoolQueryBuilder {
     }
 
     static boolean isIndexFieldNameAllowed(String fieldName) {
-        return ALLOWED_EXACT_INDEX_FIELD_NAMES.contains(fieldName)
-            || fieldName.startsWith("metadata_flattened.")
-            || fieldName.startsWith("creator.");
+        return ALLOWED_EXACT_INDEX_FIELD_NAMES.contains(fieldName) || fieldName.startsWith("metadata_flattened.");
     }
 
 }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/ApiKeyFieldNameTranslators.java

@@ -10,6 +10,8 @@ package org.elasticsearch.xpack.security.support;
 import java.util.List;
 import java.util.function.Function;
 
+import static org.elasticsearch.xpack.security.action.apikey.TransportQueryApiKeyAction.API_KEY_TYPE_RUNTIME_MAPPING_FIELD;
+
 /**
  * A class to translate query level field names to index level field names.
  */
@@ -21,6 +23,7 @@ public class ApiKeyFieldNameTranslators {
             new ExactFieldNameTranslator(s -> "creator.principal", "username"),
             new ExactFieldNameTranslator(s -> "creator.realm", "realm_name"),
             new ExactFieldNameTranslator(Function.identity(), "name"),
+            new ExactFieldNameTranslator(s -> API_KEY_TYPE_RUNTIME_MAPPING_FIELD, "type"),
             new ExactFieldNameTranslator(s -> "creation_time", "creation"),
             new ExactFieldNameTranslator(s -> "expiration_time", "expiration"),
             new ExactFieldNameTranslator(s -> "api_key_invalidated", "invalidated"),

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/ApiKeyBoolQueryBuilderTests.java

@@ -29,11 +29,13 @@ import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationTests;
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.xpack.security.action.apikey.TransportQueryApiKeyAction;
 import org.elasticsearch.xpack.security.authc.ApiKeyService;
 
 import java.io.IOException;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.function.Predicate;
 
@@ -57,7 +59,9 @@ public class ApiKeyBoolQueryBuilderTests extends ESTestCase {
     public void testBuildFromSimpleQuery() {
         final Authentication authentication = randomBoolean() ? AuthenticationTests.randomAuthentication(null, null) : null;
         final QueryBuilder q1 = randomSimpleQuery("name");
-        final ApiKeyBoolQueryBuilder apiKeyQb1 = ApiKeyBoolQueryBuilder.build(q1, authentication);
+        final List<String> queryFields = new ArrayList<>();
+        final ApiKeyBoolQueryBuilder apiKeyQb1 = ApiKeyBoolQueryBuilder.build(q1, queryFields::add, authentication);
+        assertQueryFields(queryFields, q1, authentication);
         assertCommonFilterQueries(apiKeyQb1, authentication);
         final List<QueryBuilder> mustQueries = apiKeyQb1.must();
         assertThat(mustQueries, hasSize(1));
@@ -69,7 +73,9 @@ public class ApiKeyBoolQueryBuilderTests extends ESTestCase {
     public void testQueryForDomainAuthentication() {
         final Authentication authentication = AuthenticationTests.randomAuthentication(null, AuthenticationTests.randomRealmRef(true));
         final QueryBuilder query = randomSimpleQuery("name");
-        final ApiKeyBoolQueryBuilder apiKeysQuery = ApiKeyBoolQueryBuilder.build(query, authentication);
+        final List<String> queryFields = new ArrayList<>();
+        final ApiKeyBoolQueryBuilder apiKeysQuery = ApiKeyBoolQueryBuilder.build(query, queryFields::add, authentication);
+        assertQueryFields(queryFields, query, authentication);
         assertThat(apiKeysQuery.filter().get(0), is(QueryBuilders.termQuery("doc_type", "api_key")));
         assertThat(
             apiKeysQuery.filter().get(1),
@@ -102,18 +108,23 @@ public class ApiKeyBoolQueryBuilderTests extends ESTestCase {
 
     public void testBuildFromBoolQuery() {
         final Authentication authentication = randomBoolean() ? AuthenticationTests.randomAuthentication(null, null) : null;
+        final List<String> queryFields = new ArrayList<>();
         final BoolQueryBuilder bq1 = QueryBuilders.boolQuery();
 
+        boolean accessesNameField = false;
         if (randomBoolean()) {
             bq1.must(QueryBuilders.prefixQuery("name", "prod-"));
+            accessesNameField = true;
         }
         if (randomBoolean()) {
             bq1.should(QueryBuilders.wildcardQuery("name", "*-east-*"));
+            accessesNameField = true;
         }
         if (randomBoolean()) {
             bq1.filter(
                 QueryBuilders.termsQuery("name", randomArray(3, 8, String[]::new, () -> "prod-" + randomInt() + "-east-" + randomInt()))
             );
+            accessesNameField = true;
         }
         if (randomBoolean()) {
             bq1.mustNot(QueryBuilders.idsQuery().addIds(randomArray(1, 3, String[]::new, () -> randomAlphaOfLength(22))));
@@ -121,9 +132,18 @@ public class ApiKeyBoolQueryBuilderTests extends ESTestCase {
         if (randomBoolean()) {
             bq1.minimumShouldMatch(randomIntBetween(1, 2));
         }
-        final ApiKeyBoolQueryBuilder apiKeyQb1 = ApiKeyBoolQueryBuilder.build(bq1, authentication);
+        final ApiKeyBoolQueryBuilder apiKeyQb1 = ApiKeyBoolQueryBuilder.build(bq1, queryFields::add, authentication);
         assertCommonFilterQueries(apiKeyQb1, authentication);
 
+        assertThat(queryFields, hasItem("doc_type"));
+        if (accessesNameField) {
+            assertThat(queryFields, hasItem("name"));
+        }
+        if (authentication != null && authentication.isApiKey() == false) {
+            assertThat(queryFields, hasItem("creator.principal"));
+            assertThat(queryFields, hasItem("creator.realm"));
+        }
+
         assertThat(apiKeyQb1.must(), hasSize(1));
         assertThat(apiKeyQb1.should(), empty());
         assertThat(apiKeyQb1.mustNot(), empty());
@@ -141,35 +161,78 @@ public class ApiKeyBoolQueryBuilderTests extends ESTestCase {
         final Authentication authentication = randomBoolean() ? AuthenticationTests.randomAuthentication(null, null) : null;
 
         // metadata
-        final String metadataKey = randomAlphaOfLengthBetween(3, 8);
-        final TermQueryBuilder q1 = QueryBuilders.termQuery("metadata." + metadataKey, randomAlphaOfLengthBetween(3, 8));
-        final ApiKeyBoolQueryBuilder apiKeyQb1 = ApiKeyBoolQueryBuilder.build(q1, authentication);
-        assertCommonFilterQueries(apiKeyQb1, authentication);
-        assertThat(apiKeyQb1.must().get(0), equalTo(QueryBuilders.termQuery("metadata_flattened." + metadataKey, q1.value())));
+        {
+            final List<String> queryFields = new ArrayList<>();
+            final String metadataKey = randomAlphaOfLengthBetween(3, 8);
+            final TermQueryBuilder q1 = QueryBuilders.termQuery("metadata." + metadataKey, randomAlphaOfLengthBetween(3, 8));
+            final ApiKeyBoolQueryBuilder apiKeyQb1 = ApiKeyBoolQueryBuilder.build(q1, queryFields::add, authentication);
+            assertThat(queryFields, hasItem("doc_type"));
+            assertThat(queryFields, hasItem("metadata_flattened." + metadataKey));
+            if (authentication != null && authentication.isApiKey() == false) {
+                assertThat(queryFields, hasItem("creator.principal"));
+                assertThat(queryFields, hasItem("creator.realm"));
+            }
+            assertCommonFilterQueries(apiKeyQb1, authentication);
+            assertThat(apiKeyQb1.must().get(0), equalTo(QueryBuilders.termQuery("metadata_flattened." + metadataKey, q1.value())));
+        }
 
         // username
-        final PrefixQueryBuilder q2 = QueryBuilders.prefixQuery("username", randomAlphaOfLength(3));
-        final ApiKeyBoolQueryBuilder apiKeyQb2 = ApiKeyBoolQueryBuilder.build(q2, authentication);
-        assertCommonFilterQueries(apiKeyQb2, authentication);
-        assertThat(apiKeyQb2.must().get(0), equalTo(QueryBuilders.prefixQuery("creator.principal", q2.value())));
+        {
+            final List<String> queryFields = new ArrayList<>();
+            final PrefixQueryBuilder q2 = QueryBuilders.prefixQuery("username", randomAlphaOfLength(3));
+            final ApiKeyBoolQueryBuilder apiKeyQb2 = ApiKeyBoolQueryBuilder.build(q2, queryFields::add, authentication);
+            assertThat(queryFields, hasItem("doc_type"));
+            assertThat(queryFields, hasItem("creator.principal"));
+            if (authentication != null && authentication.isApiKey() == false) {
+                assertThat(queryFields, hasItem("creator.realm"));
+            }
+            assertCommonFilterQueries(apiKeyQb2, authentication);
+            assertThat(apiKeyQb2.must().get(0), equalTo(QueryBuilders.prefixQuery("creator.principal", q2.value())));
+        }
 
         // realm name
-        final WildcardQueryBuilder q3 = QueryBuilders.wildcardQuery("realm_name", "*" + randomAlphaOfLength(3));
-        final ApiKeyBoolQueryBuilder apiKeyQb3 = ApiKeyBoolQueryBuilder.build(q3, authentication);
-        assertCommonFilterQueries(apiKeyQb3, authentication);
-        assertThat(apiKeyQb3.must().get(0), equalTo(QueryBuilders.wildcardQuery("creator.realm", q3.value())));
+        {
+            final List<String> queryFields = new ArrayList<>();
+            final WildcardQueryBuilder q3 = QueryBuilders.wildcardQuery("realm_name", "*" + randomAlphaOfLength(3));
+            final ApiKeyBoolQueryBuilder apiKeyQb3 = ApiKeyBoolQueryBuilder.build(q3, queryFields::add, authentication);
+            assertThat(queryFields, hasItem("doc_type"));
+            assertThat(queryFields, hasItem("creator.realm"));
+            if (authentication != null && authentication.isApiKey() == false) {
+                assertThat(queryFields, hasItem("creator.principal"));
+            }
+            assertCommonFilterQueries(apiKeyQb3, authentication);
+            assertThat(apiKeyQb3.must().get(0), equalTo(QueryBuilders.wildcardQuery("creator.realm", q3.value())));
+        }
 
         // creation_time
-        final TermQueryBuilder q4 = QueryBuilders.termQuery("creation", randomLongBetween(0, Long.MAX_VALUE));
-        final ApiKeyBoolQueryBuilder apiKeyQb4 = ApiKeyBoolQueryBuilder.build(q4, authentication);
-        assertCommonFilterQueries(apiKeyQb4, authentication);
-        assertThat(apiKeyQb4.must().get(0), equalTo(QueryBuilders.termQuery("creation_time", q4.value())));
+        {
+            final List<String> queryFields = new ArrayList<>();
+            final TermQueryBuilder q4 = QueryBuilders.termQuery("creation", randomLongBetween(0, Long.MAX_VALUE));
+            final ApiKeyBoolQueryBuilder apiKeyQb4 = ApiKeyBoolQueryBuilder.build(q4, queryFields::add, authentication);
+            assertThat(queryFields, hasItem("doc_type"));
+            assertThat(queryFields, hasItem("creation_time"));
+            if (authentication != null && authentication.isApiKey() == false) {
+                assertThat(queryFields, hasItem("creator.principal"));
+                assertThat(queryFields, hasItem("creator.realm"));
+            }
+            assertCommonFilterQueries(apiKeyQb4, authentication);
+            assertThat(apiKeyQb4.must().get(0), equalTo(QueryBuilders.termQuery("creation_time", q4.value())));
+        }
 
         // expiration_time
-        final TermQueryBuilder q5 = QueryBuilders.termQuery("expiration", randomLongBetween(0, Long.MAX_VALUE));
-        final ApiKeyBoolQueryBuilder apiKeyQb5 = ApiKeyBoolQueryBuilder.build(q5, authentication);
-        assertCommonFilterQueries(apiKeyQb5, authentication);
-        assertThat(apiKeyQb5.must().get(0), equalTo(QueryBuilders.termQuery("expiration_time", q5.value())));
+        {
+            final List<String> queryFields = new ArrayList<>();
+            final TermQueryBuilder q5 = QueryBuilders.termQuery("expiration", randomLongBetween(0, Long.MAX_VALUE));
+            final ApiKeyBoolQueryBuilder apiKeyQb5 = ApiKeyBoolQueryBuilder.build(q5, queryFields::add, authentication);
+            assertThat(queryFields, hasItem("doc_type"));
+            assertThat(queryFields, hasItem("expiration_time"));
+            if (authentication != null && authentication.isApiKey() == false) {
+                assertThat(queryFields, hasItem("creator.principal"));
+                assertThat(queryFields, hasItem("creator.realm"));
+            }
+            assertCommonFilterQueries(apiKeyQb5, authentication);
+            assertThat(apiKeyQb5.must().get(0), equalTo(QueryBuilders.termQuery("expiration_time", q5.value())));
+        }
     }
 
     public void testAllowListOfFieldNames() {
@@ -197,7 +260,7 @@ public class ApiKeyBoolQueryBuilderTests extends ESTestCase {
         );
         final IllegalArgumentException e1 = expectThrows(
             IllegalArgumentException.class,
-            () -> ApiKeyBoolQueryBuilder.build(q1, authentication)
+            () -> ApiKeyBoolQueryBuilder.build(q1, ignored -> {}, authentication)
         );
 
         assertThat(e1.getMessage(), containsString("Field [" + fieldName + "] is not allowed for API Key query"));
@@ -208,7 +271,7 @@ public class ApiKeyBoolQueryBuilderTests extends ESTestCase {
         final TermsQueryBuilder q1 = QueryBuilders.termsLookupQuery("name", new TermsLookup("lookup", "1", "names"));
         final IllegalArgumentException e1 = expectThrows(
             IllegalArgumentException.class,
-            () -> ApiKeyBoolQueryBuilder.build(q1, authentication)
+            () -> ApiKeyBoolQueryBuilder.build(q1, ignored -> {}, authentication)
         );
         assertThat(e1.getMessage(), containsString("terms query with terms lookup is not supported for API Key query"));
     }
@@ -218,7 +281,7 @@ public class ApiKeyBoolQueryBuilderTests extends ESTestCase {
         final RangeQueryBuilder q1 = QueryBuilders.rangeQuery("creation").relation("contains");
         final IllegalArgumentException e1 = expectThrows(
             IllegalArgumentException.class,
-            () -> ApiKeyBoolQueryBuilder.build(q1, authentication)
+            () -> ApiKeyBoolQueryBuilder.build(q1, ignored -> {}, authentication)
         );
         assertThat(e1.getMessage(), containsString("range query with relation is not supported for API Key query"));
     }
@@ -266,7 +329,7 @@ public class ApiKeyBoolQueryBuilderTests extends ESTestCase {
 
         final IllegalArgumentException e1 = expectThrows(
             IllegalArgumentException.class,
-            () -> ApiKeyBoolQueryBuilder.build(q1, authentication)
+            () -> ApiKeyBoolQueryBuilder.build(q1, ignored -> {}, authentication)
         );
         assertThat(e1.getMessage(), containsString("Query type [" + q1.getName() + "] is not supported for API Key query"));
     }
@@ -274,6 +337,7 @@ public class ApiKeyBoolQueryBuilderTests extends ESTestCase {
     public void testWillSetAllowedFields() throws IOException {
         final ApiKeyBoolQueryBuilder apiKeyQb1 = ApiKeyBoolQueryBuilder.build(
             randomSimpleQuery("name"),
+            ignored -> {},
             randomBoolean() ? AuthenticationTests.randomAuthentication(null, null) : null
         );
 
@@ -305,7 +369,11 @@ public class ApiKeyBoolQueryBuilderTests extends ESTestCase {
             new User(randomAlphaOfLengthBetween(5, 8)),
             apiKeyId
         );
-        final ApiKeyBoolQueryBuilder apiKeyQb = ApiKeyBoolQueryBuilder.build(randomFrom(randomSimpleQuery("name"), null), authentication);
+        final ApiKeyBoolQueryBuilder apiKeyQb = ApiKeyBoolQueryBuilder.build(
+            randomFrom(randomSimpleQuery("name"), null),
+            ignored -> {},
+            authentication
+        );
         assertThat(apiKeyQb.filter(), hasItem(QueryBuilders.termQuery("doc_type", "api_key")));
         assertThat(apiKeyQb.filter(), hasItem(QueryBuilders.idsQuery().addIds(apiKeyId)));
     }
@@ -314,11 +382,14 @@ public class ApiKeyBoolQueryBuilderTests extends ESTestCase {
         final String allowedField = randomFrom(
             "doc_type",
             "name",
+            "type",
+            TransportQueryApiKeyAction.API_KEY_TYPE_RUNTIME_MAPPING_FIELD,
             "api_key_invalidated",
             "creation_time",
             "expiration_time",
             "metadata_flattened." + randomAlphaOfLengthBetween(1, 10),
-            "creator." + randomAlphaOfLengthBetween(1, 10)
+            "creator.principal",
+            "creator.realm"
         );
         assertThat(predicate, trueWith(allowedField));
 
@@ -362,4 +433,15 @@ public class ApiKeyBoolQueryBuilderTests extends ESTestCase {
                 .to(Instant.now().toEpochMilli(), randomBoolean());
         };
     }
+
+    private void assertQueryFields(List<String> actualQueryFields, QueryBuilder queryBuilder, Authentication authentication) {
+        assertThat(actualQueryFields, hasItem("doc_type"));
+        if ((queryBuilder instanceof IdsQueryBuilder || queryBuilder instanceof MatchAllQueryBuilder) == false) {
+            assertThat(actualQueryFields, hasItem("name"));
+        }
+        if (authentication != null && authentication.isApiKey() == false) {
+            assertThat(actualQueryFields, hasItem("creator.principal"));
+            assertThat(actualQueryFields, hasItem("creator.realm"));
+        }
+    }
 }

x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/ApiKeyBackwardsCompatibilityIT.java

@@ -12,6 +12,7 @@ import org.apache.http.client.methods.HttpGet;
 import org.elasticsearch.Build;
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.TransportVersions;
+import org.elasticsearch.Version;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
@@ -39,19 +40,53 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.Consumer;
 
 import static org.elasticsearch.transport.RemoteClusterPortSettings.TRANSPORT_VERSION_ADVANCED_REMOTE_CLUSTER_SECURITY;
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItems;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
 
 public class ApiKeyBackwardsCompatibilityIT extends AbstractUpgradeTestCase {
 
+    private static final Version UPGRADE_FROM_VERSION = Version.fromString(System.getProperty("tests.upgrade_from_version"));
+
     private RestClient oldVersionClient = null;
     private RestClient newVersionClient = null;
 
+    public void testQueryRestTypeKeys() throws IOException {
+        assumeTrue(
+            "only API keys created pre-8.9 are relevant for the rest-type query bwc case",
+            UPGRADE_FROM_VERSION.before(Version.V_8_9_0)
+        );
+        switch (CLUSTER_TYPE) {
+            case OLD -> createOrGrantApiKey(client(), "query-test-rest-key-from-old-cluster", "{}");
+            case MIXED -> createOrGrantApiKey(client(), "query-test-rest-key-from-mixed-cluster", "{}");
+            case UPGRADED -> {
+                createOrGrantApiKey(client(), "query-test-rest-key-from-upgraded-cluster", "{}");
+                for (String query : List.of("""
+                    {"query": {"term": {"type": "rest" }}}""", """
+                    {"query": {"prefix": {"type": "re" }}}""", """
+                    {"query": {"wildcard": {"type": "r*t" }}}""", """
+                    {"query": {"range": {"type": {"gte": "raaa", "lte": "rzzz"}}}}""")) {
+                    assertQuery(client(), query, apiKeys -> {
+                        assertThat(
+                            apiKeys.stream().map(k -> (String) k.get("name")).toList(),
+                            hasItems(
+                                "query-test-rest-key-from-old-cluster",
+                                "query-test-rest-key-from-mixed-cluster",
+                                "query-test-rest-key-from-upgraded-cluster"
+                            )
+                        );
+                    });
+                }
+            }
+        }
+    }
+
     public void testCreatingAndUpdatingApiKeys() throws Exception {
         assumeTrue(
             "The remote_indices for API Keys are not supported before transport version "
@@ -177,7 +212,10 @@ public class ApiKeyBackwardsCompatibilityIT extends AbstractUpgradeTestCase {
     }
 
     private Tuple<String, String> createOrGrantApiKey(RestClient client, String roles) throws IOException {
-        final String name = "test-api-key-" + randomAlphaOfLengthBetween(3, 5);
+        return createOrGrantApiKey(client, "test-api-key-" + randomAlphaOfLengthBetween(3, 5), roles);
+    }
+
+    private Tuple<String, String> createOrGrantApiKey(RestClient client, String name, String roles) throws IOException {
         final Request createApiKeyRequest;
         String body = Strings.format("""
             {
@@ -391,4 +429,15 @@ public class ApiKeyBackwardsCompatibilityIT extends AbstractUpgradeTestCase {
             null
         );
     }
+
+    private void assertQuery(RestClient restClient, String body, Consumer<List<Map<String, Object>>> apiKeysVerifier) throws IOException {
+        final Request request = new Request("GET", "/_security/_query/api_key");
+        request.setJsonEntity(body);
+        final Response response = restClient.performRequest(request);
+        assertOK(response);
+        final Map<String, Object> responseMap = responseAsMap(response);
+        @SuppressWarnings("unchecked")
+        final List<Map<String, Object>> apiKeys = (List<Map<String, Object>>) responseMap.get("api_keys");
+        apiKeysVerifier.accept(apiKeys);
+    }
 }