Add support for API keys to access Elasticsearch (#38291)

X-Pack security supports built-in authentication service
`token-service` that allows access tokens to be used to 
access Elasticsearch without using Basic authentication.
The tokens are generated by `token-service` based on
OAuth2 spec. The access token is a short-lived token
(defaults to 20m) and refresh token with a lifetime of 24 hours,
making them unsuitable for long-lived or recurring tasks where
the system might go offline thereby failing refresh of tokens.

This commit introduces a built-in authentication service
`api-key-service` that adds support for long-lived tokens aka API
keys to access Elasticsearch. The `api-key-service` is consulted
after `token-service` in the authentication chain. By default,
if TLS is enabled then `api-key-service` is also enabled.
The service can be disabled using the configuration setting.

The API keys:-
- by default do not have an expiration but expiration can be
  configured where the API keys need to be expired after a
  certain amount of time.
- when generated will keep authentication information of the user that
   generated them.
- can be defined with a role describing the privileges for accessing
   Elasticsearch and will be limited by the role of the user that
   generated them
- can be invalidated via invalidation API
- information can be retrieved via a get API
- that have been expired or invalidated will be retained for 1 week
  before being deleted. The expired API keys remover task handles this.

Following are the API key management APIs:-
1. Create API Key - `PUT/POST /_security/api_key`
2. Get API key(s) - `GET /_security/api_key`
3. Invalidate API Key(s) `DELETE /_security/api_key`

The API keys can be used to access Elasticsearch using `Authorization`
header, where the auth scheme is `ApiKey` and the credentials, is the 
base64 encoding of API key Id and API key separated by a colon.
Example:-
```
curl -H "Authorization: ApiKey YXBpLWtleS1pZDphcGkta2V5" http://localhost:9200/_cluster/health
```

Closes #34383

client/rest-high-level/build.gradle

@@ -104,6 +104,7 @@ integTestCluster {
   setting 'xpack.license.self_generated.type', 'trial'
   setting 'xpack.security.enabled', 'true'
   setting 'xpack.security.authc.token.enabled', 'true'
+  setting 'xpack.security.authc.api_key.enabled', 'true'
   // Truststore settings are not used since TLS is not enabled. Included for testing the get certificates API
   setting 'xpack.security.http.ssl.certificate_authorities', 'testnode.crt'
   setting 'xpack.security.transport.ssl.truststore.path', 'testnode.jks'

client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityClient.java

@@ -27,6 +27,8 @@ import org.elasticsearch.client.security.ClearRealmCacheRequest;
 import org.elasticsearch.client.security.ClearRealmCacheResponse;
 import org.elasticsearch.client.security.ClearRolesCacheRequest;
 import org.elasticsearch.client.security.ClearRolesCacheResponse;
+import org.elasticsearch.client.security.CreateApiKeyRequest;
+import org.elasticsearch.client.security.CreateApiKeyResponse;
 import org.elasticsearch.client.security.CreateTokenRequest;
 import org.elasticsearch.client.security.CreateTokenResponse;
 import org.elasticsearch.client.security.DeletePrivilegesRequest;
@@ -39,6 +41,8 @@ import org.elasticsearch.client.security.DeleteUserRequest;
 import org.elasticsearch.client.security.DeleteUserResponse;
 import org.elasticsearch.client.security.DisableUserRequest;
 import org.elasticsearch.client.security.EnableUserRequest;
+import org.elasticsearch.client.security.GetApiKeyRequest;
+import org.elasticsearch.client.security.GetApiKeyResponse;
 import org.elasticsearch.client.security.GetPrivilegesRequest;
 import org.elasticsearch.client.security.GetPrivilegesResponse;
 import org.elasticsearch.client.security.GetRoleMappingsRequest;
@@ -53,6 +57,8 @@ import org.elasticsearch.client.security.GetUsersRequest;
 import org.elasticsearch.client.security.GetUsersResponse;
 import org.elasticsearch.client.security.HasPrivilegesRequest;
 import org.elasticsearch.client.security.HasPrivilegesResponse;
+import org.elasticsearch.client.security.InvalidateApiKeyRequest;
+import org.elasticsearch.client.security.InvalidateApiKeyResponse;
 import org.elasticsearch.client.security.InvalidateTokenRequest;
 import org.elasticsearch.client.security.InvalidateTokenResponse;
 import org.elasticsearch.client.security.PutPrivilegesRequest;
@@ -842,4 +848,95 @@ public final class SecurityClient {
         restHighLevelClient.performRequestAsyncAndParseEntity(request, SecurityRequestConverters::deletePrivileges, options,
             DeletePrivilegesResponse::fromXContent, listener, singleton(404));
     }
+
+    /**
+     * Create an API Key.<br>
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html">
+     * the docs</a> for more.
+     *
+     * @param request the request to create a API key
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response from the create API key call
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public CreateApiKeyResponse createApiKey(final CreateApiKeyRequest request, final RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request, SecurityRequestConverters::createApiKey, options,
+                CreateApiKeyResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Asynchronously creates an API key.<br>
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html">
+     * the docs</a> for more.
+     *
+     * @param request the request to create a API key
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void createApiKeyAsync(final CreateApiKeyRequest request, final RequestOptions options,
+            final ActionListener<CreateApiKeyResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request, SecurityRequestConverters::createApiKey, options,
+                CreateApiKeyResponse::fromXContent, listener, emptySet());
+    }
+
+    /**
+     * Retrieve API Key(s) information.<br>
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-get-api-key.html">
+     * the docs</a> for more.
+     *
+     * @param request the request to retrieve API key(s)
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response from the create API key call
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public GetApiKeyResponse getApiKey(final GetApiKeyRequest request, final RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request, SecurityRequestConverters::getApiKey, options,
+                GetApiKeyResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Asynchronously retrieve API Key(s) information.<br>
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-get-api-key.html">
+     * the docs</a> for more.
+     *
+     * @param request the request to retrieve API key(s)
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void getApiKeyAsync(final GetApiKeyRequest request, final RequestOptions options,
+            final ActionListener<GetApiKeyResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request, SecurityRequestConverters::getApiKey, options,
+                GetApiKeyResponse::fromXContent, listener, emptySet());
+    }
+
+    /**
+     * Invalidate API Key(s).<br>
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-invalidate-api-key.html">
+     * the docs</a> for more.
+     *
+     * @param request the request to invalidate API key(s)
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response from the invalidate API key call
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public InvalidateApiKeyResponse invalidateApiKey(final InvalidateApiKeyRequest request, final RequestOptions options)
+            throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request, SecurityRequestConverters::invalidateApiKey, options,
+                InvalidateApiKeyResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Asynchronously invalidates API key(s).<br>
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-invalidate-api-key.html">
+     * the docs</a> for more.
+     *
+     * @param request the request to invalidate API key(s)
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void invalidateApiKeyAsync(final InvalidateApiKeyRequest request, final RequestOptions options,
+                                      final ActionListener<InvalidateApiKeyResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request, SecurityRequestConverters::invalidateApiKey, options,
+                InvalidateApiKeyResponse::fromXContent, listener, emptySet());
+    }
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityRequestConverters.java

@@ -26,6 +26,7 @@ import org.apache.http.client.methods.HttpPut;
 import org.elasticsearch.client.security.ChangePasswordRequest;
 import org.elasticsearch.client.security.ClearRealmCacheRequest;
 import org.elasticsearch.client.security.ClearRolesCacheRequest;
+import org.elasticsearch.client.security.CreateApiKeyRequest;
 import org.elasticsearch.client.security.CreateTokenRequest;
 import org.elasticsearch.client.security.DeletePrivilegesRequest;
 import org.elasticsearch.client.security.DeleteRoleMappingRequest;
@@ -33,11 +34,13 @@ import org.elasticsearch.client.security.DeleteRoleRequest;
 import org.elasticsearch.client.security.DeleteUserRequest;
 import org.elasticsearch.client.security.DisableUserRequest;
 import org.elasticsearch.client.security.EnableUserRequest;
+import org.elasticsearch.client.security.GetApiKeyRequest;
 import org.elasticsearch.client.security.GetPrivilegesRequest;
 import org.elasticsearch.client.security.GetRoleMappingsRequest;
 import org.elasticsearch.client.security.GetRolesRequest;
 import org.elasticsearch.client.security.GetUsersRequest;
 import org.elasticsearch.client.security.HasPrivilegesRequest;
+import org.elasticsearch.client.security.InvalidateApiKeyRequest;
 import org.elasticsearch.client.security.InvalidateTokenRequest;
 import org.elasticsearch.client.security.PutPrivilegesRequest;
 import org.elasticsearch.client.security.PutRoleMappingRequest;
@@ -256,4 +259,36 @@ final class SecurityRequestConverters {
         params.withRefreshPolicy(putRoleRequest.getRefreshPolicy());
         return request;
     }
+
+    static Request createApiKey(final CreateApiKeyRequest createApiKeyRequest) throws IOException {
+        final Request request = new Request(HttpPost.METHOD_NAME, "/_security/api_key");
+        request.setEntity(createEntity(createApiKeyRequest, REQUEST_BODY_CONTENT_TYPE));
+        final RequestConverters.Params params = new RequestConverters.Params(request);
+        params.withRefreshPolicy(createApiKeyRequest.getRefreshPolicy());
+        return request;
+    }
+
+    static Request getApiKey(final GetApiKeyRequest getApiKeyRequest) throws IOException {
+        final Request request = new Request(HttpGet.METHOD_NAME, "/_security/api_key");
+        if (Strings.hasText(getApiKeyRequest.getId())) {
+            request.addParameter("id", getApiKeyRequest.getId());
+        }
+        if (Strings.hasText(getApiKeyRequest.getName())) {
+            request.addParameter("name", getApiKeyRequest.getName());
+        }
+        if (Strings.hasText(getApiKeyRequest.getUserName())) {
+            request.addParameter("username", getApiKeyRequest.getUserName());
+        }
+        if (Strings.hasText(getApiKeyRequest.getRealmName())) {
+            request.addParameter("realm_name", getApiKeyRequest.getRealmName());
+        }
+        return request;
+    }
+
+    static Request invalidateApiKey(final InvalidateApiKeyRequest invalidateApiKeyRequest) throws IOException {
+        final Request request = new Request(HttpDelete.METHOD_NAME, "/_security/api_key");
+        request.setEntity(createEntity(invalidateApiKeyRequest, REQUEST_BODY_CONTENT_TYPE));
+        final RequestConverters.Params params = new RequestConverters.Params(request);
+        return request;
+    }
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/security/CreateApiKeyRequest.java

@@ -0,0 +1,128 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.security;
+
+import org.elasticsearch.client.Validatable;
+import org.elasticsearch.client.security.user.privileges.Role;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Request to create API key
+ */
+public final class CreateApiKeyRequest implements Validatable, ToXContentObject {
+
+    private final String name;
+    private final TimeValue expiration;
+    private final List<Role> roles;
+    private final RefreshPolicy refreshPolicy;
+
+    /**
+     * Create API Key request constructor
+     * @param name name for the API key
+     * @param roles list of {@link Role}s
+     * @param expiration to specify expiration for the API key
+     */
+    public CreateApiKeyRequest(String name, List<Role> roles, @Nullable TimeValue expiration, @Nullable final RefreshPolicy refreshPolicy) {
+        if (Strings.hasText(name)) {
+            this.name = name;
+        } else {
+            throw new IllegalArgumentException("name must not be null or empty");
+        }
+        this.roles = Objects.requireNonNull(roles, "roles may not be null");
+        this.expiration = expiration;
+        this.refreshPolicy = (refreshPolicy == null) ? RefreshPolicy.getDefault() : refreshPolicy;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public TimeValue getExpiration() {
+        return expiration;
+    }
+
+    public List<Role> getRoles() {
+        return roles;
+    }
+
+    public RefreshPolicy getRefreshPolicy() {
+        return refreshPolicy;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(name, refreshPolicy, roles, expiration);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        final CreateApiKeyRequest that = (CreateApiKeyRequest) o;
+        return Objects.equals(name, that.name) && Objects.equals(refreshPolicy, that.refreshPolicy) && Objects.equals(roles, that.roles)
+                && Objects.equals(expiration, that.expiration);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject().field("name", name);
+        if (expiration != null) {
+            builder.field("expiration", expiration.getStringRep());
+        }
+        builder.startObject("role_descriptors");
+        for (Role role : roles) {
+            builder.startObject(role.getName());
+            if (role.getApplicationPrivileges() != null) {
+                builder.field(Role.APPLICATIONS.getPreferredName(), role.getApplicationPrivileges());
+            }
+            if (role.getClusterPrivileges() != null) {
+                builder.field(Role.CLUSTER.getPreferredName(), role.getClusterPrivileges());
+            }
+            if (role.getGlobalPrivileges() != null) {
+                builder.field(Role.GLOBAL.getPreferredName(), role.getGlobalPrivileges());
+            }
+            if (role.getIndicesPrivileges() != null) {
+                builder.field(Role.INDICES.getPreferredName(), role.getIndicesPrivileges());
+            }
+            if (role.getMetadata() != null) {
+                builder.field(Role.METADATA.getPreferredName(), role.getMetadata());
+            }
+            if (role.getRunAsPrivilege() != null) {
+                builder.field(Role.RUN_AS.getPreferredName(), role.getRunAsPrivilege());
+            }
+            builder.endObject();
+        }
+        builder.endObject();
+        return builder.endObject();
+    }
+
+}

client/rest-high-level/src/main/java/org/elasticsearch/client/security/CreateApiKeyResponse.java

@@ -0,0 +1,105 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.security;
+
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
+/**
+ * Response for create API key
+ */
+public final class CreateApiKeyResponse {
+
+    private final String name;
+    private final String id;
+    private final SecureString key;
+    private final Instant expiration;
+
+    public CreateApiKeyResponse(String name, String id, SecureString key, Instant expiration) {
+        this.name = name;
+        this.id = id;
+        this.key = key;
+        // As we do not yet support the nanosecond precision when we serialize to JSON,
+        // here creating the 'Instant' of milliseconds precision.
+        // This Instant can then be used for date comparison.
+        this.expiration = (expiration != null) ? Instant.ofEpochMilli(expiration.toEpochMilli()): null;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public String getId() {
+        return id;
+    }
+
+    public SecureString getKey() {
+        return key;
+    }
+
+    @Nullable
+    public Instant getExpiration() {
+        return expiration;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(id, name, key, expiration);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+        final CreateApiKeyResponse other = (CreateApiKeyResponse) obj;
+        return Objects.equals(id, other.id)
+                && Objects.equals(key, other.key)
+                && Objects.equals(name, other.name)
+                && Objects.equals(expiration, other.expiration);
+    }
+
+    static ConstructingObjectParser<CreateApiKeyResponse, Void> PARSER = new ConstructingObjectParser<>("create_api_key_response",
+            args -> new CreateApiKeyResponse((String) args[0], (String) args[1], new SecureString((String) args[2]),
+                    (args[3] == null) ? null : Instant.ofEpochMilli((Long) args[3])));
+    static {
+        PARSER.declareString(constructorArg(), new ParseField("name"));
+        PARSER.declareString(constructorArg(), new ParseField("id"));
+        PARSER.declareString(constructorArg(), new ParseField("api_key"));
+        PARSER.declareLong(optionalConstructorArg(), new ParseField("expiration"));
+    }
+
+    public static CreateApiKeyResponse fromXContent(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
+}

client/rest-high-level/src/main/java/org/elasticsearch/client/security/GetApiKeyRequest.java

@@ -0,0 +1,133 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.security;
+
+import org.elasticsearch.client.Validatable;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+
+/**
+ * Request for get API key
+ */
+public final class GetApiKeyRequest implements Validatable, ToXContentObject {
+
+    private final String realmName;
+    private final String userName;
+    private final String id;
+    private final String name;
+
+    // pkg scope for testing
+    GetApiKeyRequest(@Nullable String realmName, @Nullable String userName, @Nullable String apiKeyId,
+            @Nullable String apiKeyName) {
+        if (Strings.hasText(realmName) == false && Strings.hasText(userName) == false && Strings.hasText(apiKeyId) == false
+                && Strings.hasText(apiKeyName) == false) {
+            throwValidationError("One of [api key id, api key name, username, realm name] must be specified");
+        }
+        if (Strings.hasText(apiKeyId) || Strings.hasText(apiKeyName)) {
+            if (Strings.hasText(realmName) || Strings.hasText(userName)) {
+                throwValidationError(
+                        "username or realm name must not be specified when the api key id or api key name is specified");
+            }
+        }
+        if (Strings.hasText(apiKeyId) && Strings.hasText(apiKeyName)) {
+            throwValidationError("only one of [api key id, api key name] can be specified");
+        }
+        this.realmName = realmName;
+        this.userName = userName;
+        this.id = apiKeyId;
+        this.name = apiKeyName;
+    }
+
+    private void throwValidationError(String message) {
+        throw new IllegalArgumentException(message);
+    }
+
+    public String getRealmName() {
+        return realmName;
+    }
+
+    public String getUserName() {
+        return userName;
+    }
+
+    public String getId() {
+        return id;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    /**
+     * Creates get API key request for given realm name
+     * @param realmName realm name
+     * @return {@link GetApiKeyRequest}
+     */
+    public static GetApiKeyRequest usingRealmName(String realmName) {
+        return new GetApiKeyRequest(realmName, null, null, null);
+    }
+
+    /**
+     * Creates get API key request for given user name
+     * @param userName user name
+     * @return {@link GetApiKeyRequest}
+     */
+    public static GetApiKeyRequest usingUserName(String userName) {
+        return new GetApiKeyRequest(null, userName, null, null);
+    }
+
+    /**
+     * Creates get API key request for given realm and user name
+     * @param realmName realm name
+     * @param userName user name
+     * @return {@link GetApiKeyRequest}
+     */
+    public static GetApiKeyRequest usingRealmAndUserName(String realmName, String userName) {
+        return new GetApiKeyRequest(realmName, userName, null, null);
+    }
+
+    /**
+     * Creates get API key request for given api key id
+     * @param apiKeyId api key id
+     * @return {@link GetApiKeyRequest}
+     */
+    public static GetApiKeyRequest usingApiKeyId(String apiKeyId) {
+        return new GetApiKeyRequest(null, null, apiKeyId, null);
+    }
+
+    /**
+     * Creates get API key request for given api key name
+     * @param apiKeyName api key name
+     * @return {@link GetApiKeyRequest}
+     */
+    public static GetApiKeyRequest usingApiKeyName(String apiKeyName) {
+        return new GetApiKeyRequest(null, null, null, apiKeyName);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        return builder;
+    }
+
+}

client/rest-high-level/src/main/java/org/elasticsearch/client/security/GetApiKeyResponse.java

@@ -0,0 +1,91 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.security;
+
+import org.elasticsearch.client.security.support.ApiKey;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
+/**
+ * Response for get API keys.<br>
+ * The result contains information about the API keys that were found.
+ */
+public final class GetApiKeyResponse {
+
+    private final List<ApiKey> foundApiKeysInfo;
+
+    public GetApiKeyResponse(List<ApiKey> foundApiKeysInfo) {
+        Objects.requireNonNull(foundApiKeysInfo, "found_api_keys_info must be provided");
+        this.foundApiKeysInfo = Collections.unmodifiableList(foundApiKeysInfo);
+    }
+
+    public static GetApiKeyResponse emptyResponse() {
+        return new GetApiKeyResponse(Collections.emptyList());
+    }
+
+    public List<ApiKey> getApiKeyInfos() {
+        return foundApiKeysInfo;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(foundApiKeysInfo);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null) {
+            return false;
+        }
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+        final GetApiKeyResponse other = (GetApiKeyResponse) obj;
+        return Objects.equals(foundApiKeysInfo, other.foundApiKeysInfo);
+    }
+
+    @SuppressWarnings("unchecked")
+    static ConstructingObjectParser<GetApiKeyResponse, Void> PARSER = new ConstructingObjectParser<>("get_api_key_response", args -> {
+        return (args[0] == null) ? GetApiKeyResponse.emptyResponse() : new GetApiKeyResponse((List<ApiKey>) args[0]);
+    });
+    static {
+        PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> ApiKey.fromXContent(p), new ParseField("api_keys"));
+    }
+
+    public static GetApiKeyResponse fromXContent(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
+
+    @Override
+    public String toString() {
+        return "GetApiKeyResponse [foundApiKeysInfo=" + foundApiKeysInfo + "]";
+    }
+}

client/rest-high-level/src/main/java/org/elasticsearch/client/security/InvalidateApiKeyRequest.java

@@ -0,0 +1,145 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.security;
+
+import org.elasticsearch.client.Validatable;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+
+/**
+ * Request for invalidating API key(s) so that it can no longer be used
+ */
+public final class InvalidateApiKeyRequest implements Validatable, ToXContentObject {
+
+    private final String realmName;
+    private final String userName;
+    private final String id;
+    private final String name;
+
+    // pkg scope for testing
+    InvalidateApiKeyRequest(@Nullable String realmName, @Nullable String userName, @Nullable String apiKeyId,
+            @Nullable String apiKeyName) {
+        if (Strings.hasText(realmName) == false && Strings.hasText(userName) == false && Strings.hasText(apiKeyId) == false
+                && Strings.hasText(apiKeyName) == false) {
+            throwValidationError("One of [api key id, api key name, username, realm name] must be specified");
+        }
+        if (Strings.hasText(apiKeyId) || Strings.hasText(apiKeyName)) {
+            if (Strings.hasText(realmName) || Strings.hasText(userName)) {
+                throwValidationError(
+                        "username or realm name must not be specified when the api key id or api key name is specified");
+            }
+        }
+        if (Strings.hasText(apiKeyId) && Strings.hasText(apiKeyName)) {
+            throwValidationError("only one of [api key id, api key name] can be specified");
+        }
+        this.realmName = realmName;
+        this.userName = userName;
+        this.id = apiKeyId;
+        this.name = apiKeyName;
+    }
+
+    private void throwValidationError(String message) {
+        throw new IllegalArgumentException(message);
+    }
+
+    public String getRealmName() {
+        return realmName;
+    }
+
+    public String getUserName() {
+        return userName;
+    }
+
+    public String getId() {
+        return id;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    /**
+     * Creates invalidate API key request for given realm name
+     * @param realmName realm name
+     * @return {@link InvalidateApiKeyRequest}
+     */
+    public static InvalidateApiKeyRequest usingRealmName(String realmName) {
+        return new InvalidateApiKeyRequest(realmName, null, null, null);
+    }
+
+    /**
+     * Creates invalidate API key request for given user name
+     * @param userName user name
+     * @return {@link InvalidateApiKeyRequest}
+     */
+    public static InvalidateApiKeyRequest usingUserName(String userName) {
+        return new InvalidateApiKeyRequest(null, userName, null, null);
+    }
+
+    /**
+     * Creates invalidate API key request for given realm and user name
+     * @param realmName realm name
+     * @param userName user name
+     * @return {@link InvalidateApiKeyRequest}
+     */
+    public static InvalidateApiKeyRequest usingRealmAndUserName(String realmName, String userName) {
+        return new InvalidateApiKeyRequest(realmName, userName, null, null);
+    }
+
+    /**
+     * Creates invalidate API key request for given api key id
+     * @param apiKeyId api key id
+     * @return {@link InvalidateApiKeyRequest}
+     */
+    public static InvalidateApiKeyRequest usingApiKeyId(String apiKeyId) {
+        return new InvalidateApiKeyRequest(null, null, apiKeyId, null);
+    }
+
+    /**
+     * Creates invalidate API key request for given api key name
+     * @param apiKeyName api key name
+     * @return {@link InvalidateApiKeyRequest}
+     */
+    public static InvalidateApiKeyRequest usingApiKeyName(String apiKeyName) {
+        return new InvalidateApiKeyRequest(null, null, null, apiKeyName);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        if (realmName != null) {
+            builder.field("realm_name", realmName);
+        }
+        if (userName != null) {
+            builder.field("username", userName);
+        }
+        if (id != null) {
+            builder.field("id", id);
+        }
+        if (name != null) {
+            builder.field("name", name);
+        }
+        return builder.endObject();
+    }
+}

client/rest-high-level/src/main/java/org/elasticsearch/client/security/InvalidateApiKeyResponse.java

@@ -0,0 +1,121 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.security;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
+public final class InvalidateApiKeyResponse {
+
+    private final List<String> invalidatedApiKeys;
+    private final List<String> previouslyInvalidatedApiKeys;
+    private final List<ElasticsearchException> errors;
+
+    /**
+     * Constructor for API keys invalidation response
+     * @param invalidatedApiKeys list of invalidated API key ids
+     * @param previouslyInvalidatedApiKeys list of previously invalidated API key ids
+     * @param errors list of encountered errors while invalidating API keys
+     */
+    public InvalidateApiKeyResponse(List<String> invalidatedApiKeys, List<String> previouslyInvalidatedApiKeys,
+                                    @Nullable List<ElasticsearchException> errors) {
+        this.invalidatedApiKeys = Objects.requireNonNull(invalidatedApiKeys, "invalidated_api_keys must be provided");
+        this.previouslyInvalidatedApiKeys = Objects.requireNonNull(previouslyInvalidatedApiKeys,
+                "previously_invalidated_api_keys must be provided");
+        if (null != errors) {
+            this.errors = errors;
+        } else {
+            this.errors = Collections.emptyList();
+        }
+    }
+
+    public static InvalidateApiKeyResponse emptyResponse() {
+        return new InvalidateApiKeyResponse(Collections.emptyList(), Collections.emptyList(), Collections.emptyList());
+    }
+
+    public List<String> getInvalidatedApiKeys() {
+        return invalidatedApiKeys;
+    }
+
+    public List<String> getPreviouslyInvalidatedApiKeys() {
+        return previouslyInvalidatedApiKeys;
+    }
+
+    public List<ElasticsearchException> getErrors() {
+        return errors;
+    }
+
+    @SuppressWarnings("unchecked")
+    static ConstructingObjectParser<InvalidateApiKeyResponse, Void> PARSER = new ConstructingObjectParser<>("invalidate_api_key_response",
+            args -> {
+                return new InvalidateApiKeyResponse((List<String>) args[0], (List<String>) args[1], (List<ElasticsearchException>) args[3]);
+            });
+    static {
+        PARSER.declareStringArray(constructorArg(), new ParseField("invalidated_api_keys"));
+        PARSER.declareStringArray(constructorArg(), new ParseField("previously_invalidated_api_keys"));
+        // error count is parsed but ignored as we have list of errors
+        PARSER.declareInt(constructorArg(), new ParseField("error_count"));
+        PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> ElasticsearchException.fromXContent(p),
+                new ParseField("error_details"));
+    }
+
+    public static InvalidateApiKeyResponse fromXContent(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(invalidatedApiKeys, previouslyInvalidatedApiKeys, errors);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null) {
+            return false;
+        }
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+        InvalidateApiKeyResponse other = (InvalidateApiKeyResponse) obj;
+        return Objects.equals(invalidatedApiKeys, other.invalidatedApiKeys)
+                && Objects.equals(previouslyInvalidatedApiKeys, other.previouslyInvalidatedApiKeys)
+                && Objects.equals(errors, other.errors);
+    }
+
+    @Override
+    public String toString() {
+        return "ApiKeysInvalidationResult [invalidatedApiKeys=" + invalidatedApiKeys + ", previouslyInvalidatedApiKeys="
+                + previouslyInvalidatedApiKeys + ", errors=" + errors + "]";
+    }
+}

client/rest-high-level/src/main/java/org/elasticsearch/client/security/support/ApiKey.java

@@ -0,0 +1,152 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.security.support;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
+/**
+ * API key information
+ */
+public final class ApiKey {
+
+    private final String name;
+    private final String id;
+    private final Instant creation;
+    private final Instant expiration;
+    private final boolean invalidated;
+    private final String username;
+    private final String realm;
+
+    public ApiKey(String name, String id, Instant creation, Instant expiration, boolean invalidated, String username, String realm) {
+        this.name = name;
+        this.id = id;
+        // As we do not yet support the nanosecond precision when we serialize to JSON,
+        // here creating the 'Instant' of milliseconds precision.
+        // This Instant can then be used for date comparison.
+        this.creation = Instant.ofEpochMilli(creation.toEpochMilli());
+        this.expiration = (expiration != null) ? Instant.ofEpochMilli(expiration.toEpochMilli()): null;
+        this.invalidated = invalidated;
+        this.username = username;
+        this.realm = realm;
+    }
+
+    public String getId() {
+        return id;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    /**
+     * @return a instance of {@link Instant} when this API key was created.
+     */
+    public Instant getCreation() {
+        return creation;
+    }
+
+    /**
+     * @return a instance of {@link Instant} when this API key will expire. In case the API key does not expire then will return
+     * {@code null}
+     */
+    public Instant getExpiration() {
+        return expiration;
+    }
+
+    /**
+     * @return {@code true} if this API key has been invalidated else returns {@code false}
+     */
+    public boolean isInvalidated() {
+        return invalidated;
+    }
+
+    /**
+     * @return the username for which this API key was created.
+     */
+    public String getUsername() {
+        return username;
+    }
+
+    /**
+     * @return the realm name of the user for which this API key was created.
+     */
+    public String getRealm() {
+        return realm;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(name, id, creation, expiration, invalidated, username, realm);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null) {
+            return false;
+        }
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+        ApiKey other = (ApiKey) obj;
+        return Objects.equals(name, other.name)
+                && Objects.equals(id, other.id)
+                && Objects.equals(creation, other.creation)
+                && Objects.equals(expiration, other.expiration)
+                && Objects.equals(invalidated, other.invalidated)
+                && Objects.equals(username, other.username)
+                && Objects.equals(realm, other.realm);
+    }
+
+    static ConstructingObjectParser<ApiKey, Void> PARSER = new ConstructingObjectParser<>("api_key", args -> {
+        return new ApiKey((String) args[0], (String) args[1], Instant.ofEpochMilli((Long) args[2]),
+                (args[3] == null) ? null : Instant.ofEpochMilli((Long) args[3]), (Boolean) args[4], (String) args[5], (String) args[6]);
+    });
+    static {
+        PARSER.declareString(constructorArg(), new ParseField("name"));
+        PARSER.declareString(constructorArg(), new ParseField("id"));
+        PARSER.declareLong(constructorArg(), new ParseField("creation"));
+        PARSER.declareLong(optionalConstructorArg(), new ParseField("expiration"));
+        PARSER.declareBoolean(constructorArg(), new ParseField("invalidated"));
+        PARSER.declareString(constructorArg(), new ParseField("username"));
+        PARSER.declareString(constructorArg(), new ParseField("realm"));
+    }
+
+    public static ApiKey fromXContent(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
+
+    @Override
+    public String toString() {
+        return "ApiKey [name=" + name + ", id=" + id + ", creation=" + creation + ", expiration=" + expiration + ", invalidated="
+                + invalidated + ", username=" + username + ", realm=" + realm + "]";
+    }
+}

client/rest-high-level/src/test/java/org/elasticsearch/client/SecurityRequestConvertersTests.java

@@ -24,6 +24,7 @@ import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpPut;
 import org.elasticsearch.client.security.ChangePasswordRequest;
+import org.elasticsearch.client.security.CreateApiKeyRequest;
 import org.elasticsearch.client.security.CreateTokenRequest;
 import org.elasticsearch.client.security.DeletePrivilegesRequest;
 import org.elasticsearch.client.security.DeleteRoleMappingRequest;
@@ -31,10 +32,12 @@ import org.elasticsearch.client.security.DeleteRoleRequest;
 import org.elasticsearch.client.security.DeleteUserRequest;
 import org.elasticsearch.client.security.DisableUserRequest;
 import org.elasticsearch.client.security.EnableUserRequest;
+import org.elasticsearch.client.security.GetApiKeyRequest;
 import org.elasticsearch.client.security.GetPrivilegesRequest;
 import org.elasticsearch.client.security.GetRoleMappingsRequest;
 import org.elasticsearch.client.security.GetRolesRequest;
 import org.elasticsearch.client.security.GetUsersRequest;
+import org.elasticsearch.client.security.InvalidateApiKeyRequest;
 import org.elasticsearch.client.security.PutPrivilegesRequest;
 import org.elasticsearch.client.security.PutRoleMappingRequest;
 import org.elasticsearch.client.security.PutRoleRequest;
@@ -44,11 +47,14 @@ import org.elasticsearch.client.security.support.expressiondsl.RoleMapperExpress
 import org.elasticsearch.client.security.support.expressiondsl.expressions.AnyRoleMapperExpression;
 import org.elasticsearch.client.security.support.expressiondsl.fields.FieldRoleMapperExpression;
 import org.elasticsearch.client.security.user.User;
-import org.elasticsearch.client.security.user.privileges.ApplicationResourcePrivileges;
 import org.elasticsearch.client.security.user.privileges.ApplicationPrivilege;
+import org.elasticsearch.client.security.user.privileges.ApplicationResourcePrivileges;
 import org.elasticsearch.client.security.user.privileges.IndicesPrivileges;
 import org.elasticsearch.client.security.user.privileges.Role;
+import org.elasticsearch.client.security.user.privileges.Role.ClusterPrivilegeName;
+import org.elasticsearch.client.security.user.privileges.Role.IndexPrivilegeName;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.test.ESTestCase;
 
@@ -61,6 +67,7 @@ import java.util.List;
 import java.util.Map;
 
 import static org.elasticsearch.client.RequestConvertersTests.assertToXContentBody;
+import static org.hamcrest.Matchers.equalTo;
 
 public class SecurityRequestConvertersTests extends ESTestCase {
 
@@ -411,4 +418,47 @@ public class SecurityRequestConvertersTests extends ESTestCase {
         assertEquals(expectedParams, request.getParameters());
         assertToXContentBody(putRoleRequest, request.getEntity());
     }
-}
+
+    public void testCreateApiKey() throws IOException {
+        final String name = randomAlphaOfLengthBetween(4, 7);
+        final List<Role> roles = Collections.singletonList(Role.builder().name("r1").clusterPrivileges(ClusterPrivilegeName.ALL)
+                .indicesPrivileges(IndicesPrivileges.builder().indices("ind-x").privileges(IndexPrivilegeName.ALL).build()).build());
+        final TimeValue expiration = randomBoolean() ? null : TimeValue.timeValueHours(24);
+        final RefreshPolicy refreshPolicy = randomFrom(RefreshPolicy.values());
+        final Map<String, String> expectedParams;
+        if (refreshPolicy != RefreshPolicy.NONE) {
+            expectedParams = Collections.singletonMap("refresh", refreshPolicy.getValue());
+        } else {
+            expectedParams = Collections.emptyMap();
+        }
+        final CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest(name, roles, expiration, refreshPolicy);
+        final Request request = SecurityRequestConverters.createApiKey(createApiKeyRequest);
+        assertEquals(HttpPost.METHOD_NAME, request.getMethod());
+        assertEquals("/_security/api_key", request.getEndpoint());
+        assertEquals(expectedParams, request.getParameters());
+        assertToXContentBody(createApiKeyRequest, request.getEntity());
+    }
+
+    public void testGetApiKey() throws IOException {
+        String realmName = randomAlphaOfLength(5);
+        String userName = randomAlphaOfLength(7);
+        final GetApiKeyRequest getApiKeyRequest = GetApiKeyRequest.usingRealmAndUserName(realmName, userName);
+        final Request request = SecurityRequestConverters.getApiKey(getApiKeyRequest);
+        assertEquals(HttpGet.METHOD_NAME, request.getMethod());
+        assertEquals("/_security/api_key", request.getEndpoint());
+        Map<String, String> mapOfParameters = new HashMap<>();
+        mapOfParameters.put("realm_name", realmName);
+        mapOfParameters.put("username", userName);
+        assertThat(request.getParameters(), equalTo(mapOfParameters));
+    }
+
+    public void testInvalidateApiKey() throws IOException {
+        String realmName = randomAlphaOfLength(5);
+        String userName = randomAlphaOfLength(7);
+        final InvalidateApiKeyRequest invalidateApiKeyRequest = InvalidateApiKeyRequest.usingRealmAndUserName(realmName, userName);
+        final Request request = SecurityRequestConverters.invalidateApiKey(invalidateApiKeyRequest);
+        assertEquals(HttpDelete.METHOD_NAME, request.getMethod());
+        assertEquals("/_security/api_key", request.getEndpoint());
+        assertToXContentBody(invalidateApiKeyRequest, request.getEntity());
+    }
+ }

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java

@@ -33,6 +33,8 @@ import org.elasticsearch.client.security.ClearRealmCacheRequest;
 import org.elasticsearch.client.security.ClearRealmCacheResponse;
 import org.elasticsearch.client.security.ClearRolesCacheRequest;
 import org.elasticsearch.client.security.ClearRolesCacheResponse;
+import org.elasticsearch.client.security.CreateApiKeyRequest;
+import org.elasticsearch.client.security.CreateApiKeyResponse;
 import org.elasticsearch.client.security.CreateTokenRequest;
 import org.elasticsearch.client.security.CreateTokenResponse;
 import org.elasticsearch.client.security.DeletePrivilegesRequest;
@@ -46,6 +48,8 @@ import org.elasticsearch.client.security.DeleteUserResponse;
 import org.elasticsearch.client.security.DisableUserRequest;
 import org.elasticsearch.client.security.EnableUserRequest;
 import org.elasticsearch.client.security.ExpressionRoleMapping;
+import org.elasticsearch.client.security.GetApiKeyRequest;
+import org.elasticsearch.client.security.GetApiKeyResponse;
 import org.elasticsearch.client.security.GetPrivilegesRequest;
 import org.elasticsearch.client.security.GetPrivilegesResponse;
 import org.elasticsearch.client.security.GetRoleMappingsRequest;
@@ -58,6 +62,8 @@ import org.elasticsearch.client.security.GetUsersRequest;
 import org.elasticsearch.client.security.GetUsersResponse;
 import org.elasticsearch.client.security.HasPrivilegesRequest;
 import org.elasticsearch.client.security.HasPrivilegesResponse;
+import org.elasticsearch.client.security.InvalidateApiKeyRequest;
+import org.elasticsearch.client.security.InvalidateApiKeyResponse;
 import org.elasticsearch.client.security.InvalidateTokenRequest;
 import org.elasticsearch.client.security.InvalidateTokenResponse;
 import org.elasticsearch.client.security.PutPrivilegesRequest;
@@ -69,6 +75,7 @@ import org.elasticsearch.client.security.PutRoleResponse;
 import org.elasticsearch.client.security.PutUserRequest;
 import org.elasticsearch.client.security.PutUserResponse;
 import org.elasticsearch.client.security.RefreshPolicy;
+import org.elasticsearch.client.security.support.ApiKey;
 import org.elasticsearch.client.security.support.CertificateInfo;
 import org.elasticsearch.client.security.support.expressiondsl.RoleMapperExpression;
 import org.elasticsearch.client.security.support.expressiondsl.expressions.AnyRoleMapperExpression;
@@ -78,13 +85,17 @@ import org.elasticsearch.client.security.user.privileges.ApplicationPrivilege;
 import org.elasticsearch.client.security.user.privileges.ApplicationResourcePrivileges;
 import org.elasticsearch.client.security.user.privileges.IndicesPrivileges;
 import org.elasticsearch.client.security.user.privileges.Role;
+import org.elasticsearch.client.security.user.privileges.Role.ClusterPrivilegeName;
+import org.elasticsearch.client.security.user.privileges.Role.IndexPrivilegeName;
 import org.elasticsearch.client.security.user.privileges.UserIndicesPrivileges;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.set.Sets;
 import org.hamcrest.Matchers;
 
-import javax.crypto.SecretKeyFactory;
-import javax.crypto.spec.PBEKeySpec;
 import java.io.IOException;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Base64;
@@ -97,15 +108,20 @@ import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
+import javax.crypto.SecretKeyFactory;
+import javax.crypto.spec.PBEKeySpec;
+
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.emptyIterable;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.isIn;
 import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
 public class SecurityDocumentationIT extends ESRestHighLevelClientTestCase {
@@ -336,7 +352,7 @@ public class SecurityDocumentationIT extends ESRestHighLevelClientTestCase {
 
     private void addUser(RestHighLevelClient client, String userName, String password) throws IOException {
         User user = new User(userName, Collections.singletonList(userName));
-        PutUserRequest request = new PutUserRequest(user, password.toCharArray(), true, RefreshPolicy.NONE);
+        PutUserRequest request = PutUserRequest.withPassword(user, password.toCharArray(), true, RefreshPolicy.NONE);
         PutUserResponse response = client.security().putUser(request, RequestOptions.DEFAULT);
         assertTrue(response.isCreated());
     }
@@ -510,7 +526,7 @@ public class SecurityDocumentationIT extends ESRestHighLevelClientTestCase {
         RestHighLevelClient client = highLevelClient();
         char[] password = new char[]{'p', 'a', 's', 's', 'w', 'o', 'r', 'd'};
         User enable_user = new User("enable_user", Collections.singletonList("superuser"));
-        PutUserRequest putUserRequest = new PutUserRequest(enable_user, password, true, RefreshPolicy.IMMEDIATE);
+        PutUserRequest putUserRequest = PutUserRequest.withPassword(enable_user, password, true, RefreshPolicy.IMMEDIATE);
         PutUserResponse putUserResponse = client.security().putUser(putUserRequest, RequestOptions.DEFAULT);
         assertTrue(putUserResponse.isCreated());
 
@@ -555,7 +571,7 @@ public class SecurityDocumentationIT extends ESRestHighLevelClientTestCase {
         RestHighLevelClient client = highLevelClient();
         char[] password = new char[]{'p', 'a', 's', 's', 'w', 'o', 'r', 'd'};
         User disable_user = new User("disable_user", Collections.singletonList("superuser"));
-        PutUserRequest putUserRequest = new PutUserRequest(disable_user, password, true, RefreshPolicy.IMMEDIATE);
+        PutUserRequest putUserRequest = PutUserRequest.withPassword(disable_user, password, true, RefreshPolicy.IMMEDIATE);
         PutUserResponse putUserResponse = client.security().putUser(putUserRequest, RequestOptions.DEFAULT);
         assertTrue(putUserResponse.isCreated());
         {
@@ -1032,7 +1048,7 @@ public class SecurityDocumentationIT extends ESRestHighLevelClientTestCase {
         char[] password = new char[]{'p', 'a', 's', 's', 'w', 'o', 'r', 'd'};
         char[] newPassword = new char[]{'n', 'e', 'w', 'p', 'a', 's', 's', 'w', 'o', 'r', 'd'};
         User user = new User("change_password_user", Collections.singletonList("superuser"), Collections.emptyMap(), null, null);
-        PutUserRequest putUserRequest = new PutUserRequest(user, password, true, RefreshPolicy.NONE);
+        PutUserRequest putUserRequest = PutUserRequest.withPassword(user, password, true, RefreshPolicy.NONE);
         PutUserResponse putUserResponse = client.security().putUser(putUserRequest, RequestOptions.DEFAULT);
         assertTrue(putUserResponse.isCreated());
         {
@@ -1249,7 +1265,8 @@ public class SecurityDocumentationIT extends ESRestHighLevelClientTestCase {
         {
             // Setup user
             User token_user = new User("token_user", Collections.singletonList("kibana_user"));
-            PutUserRequest putUserRequest = new PutUserRequest(token_user, "password".toCharArray(), true, RefreshPolicy.IMMEDIATE);
+            PutUserRequest putUserRequest = PutUserRequest.withPassword(token_user, "password".toCharArray(), true,
+                    RefreshPolicy.IMMEDIATE);
             PutUserResponse putUserResponse = client.security().putUser(putUserRequest, RequestOptions.DEFAULT);
             assertTrue(putUserResponse.isCreated());
         }
@@ -1327,27 +1344,27 @@ public class SecurityDocumentationIT extends ESRestHighLevelClientTestCase {
             // Setup users
             final char[] password = "password".toCharArray();
             User user = new User("user", Collections.singletonList("kibana_user"));
-            PutUserRequest putUserRequest = new PutUserRequest(user, password, true, RefreshPolicy.IMMEDIATE);
+            PutUserRequest putUserRequest = PutUserRequest.withPassword(user, password, true, RefreshPolicy.IMMEDIATE);
             PutUserResponse putUserResponse = client.security().putUser(putUserRequest, RequestOptions.DEFAULT);
             assertTrue(putUserResponse.isCreated());
 
             User this_user = new User("this_user", Collections.singletonList("kibana_user"));
-            PutUserRequest putThisUserRequest = new PutUserRequest(this_user, password, true, RefreshPolicy.IMMEDIATE);
+            PutUserRequest putThisUserRequest = PutUserRequest.withPassword(this_user, password, true, RefreshPolicy.IMMEDIATE);
             PutUserResponse putThisUserResponse = client.security().putUser(putThisUserRequest, RequestOptions.DEFAULT);
             assertTrue(putThisUserResponse.isCreated());
 
             User that_user = new User("that_user", Collections.singletonList("kibana_user"));
-            PutUserRequest putThatUserRequest = new PutUserRequest(that_user, password, true, RefreshPolicy.IMMEDIATE);
+            PutUserRequest putThatUserRequest = PutUserRequest.withPassword(that_user, password, true, RefreshPolicy.IMMEDIATE);
             PutUserResponse putThatUserResponse = client.security().putUser(putThatUserRequest, RequestOptions.DEFAULT);
             assertTrue(putThatUserResponse.isCreated());
 
             User other_user = new User("other_user", Collections.singletonList("kibana_user"));
-            PutUserRequest putOtherUserRequest = new PutUserRequest(other_user, password, true, RefreshPolicy.IMMEDIATE);
+            PutUserRequest putOtherUserRequest = PutUserRequest.withPassword(other_user, password, true, RefreshPolicy.IMMEDIATE);
             PutUserResponse putOtherUserResponse = client.security().putUser(putOtherUserRequest, RequestOptions.DEFAULT);
             assertTrue(putOtherUserResponse.isCreated());
 
             User extra_user = new User("extra_user", Collections.singletonList("kibana_user"));
-            PutUserRequest putExtraUserRequest = new PutUserRequest(extra_user, password, true, RefreshPolicy.IMMEDIATE);
+            PutUserRequest putExtraUserRequest = PutUserRequest.withPassword(extra_user, password, true, RefreshPolicy.IMMEDIATE);
             PutUserResponse putExtraUserResponse = client.security().putUser(putExtraUserRequest, RequestOptions.DEFAULT);
             assertTrue(putExtraUserResponse.isCreated());
 
@@ -1747,4 +1764,363 @@ public class SecurityDocumentationIT extends ESRestHighLevelClientTestCase {
         }
     }
 
+    public void testCreateApiKey() throws Exception {
+        RestHighLevelClient client = highLevelClient();
+
+        List<Role> roles = Collections.singletonList(Role.builder().name("r1").clusterPrivileges(ClusterPrivilegeName.ALL)
+                .indicesPrivileges(IndicesPrivileges.builder().indices("ind-x").privileges(IndexPrivilegeName.ALL).build()).build());
+        final TimeValue expiration = TimeValue.timeValueHours(24);
+        final RefreshPolicy refreshPolicy = randomFrom(RefreshPolicy.values());
+        {
+            final String name = randomAlphaOfLength(5);
+            // tag::create-api-key-request
+            CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest(name, roles, expiration, refreshPolicy);
+            // end::create-api-key-request
+
+            // tag::create-api-key-execute
+            CreateApiKeyResponse createApiKeyResponse = client.security().createApiKey(createApiKeyRequest, RequestOptions.DEFAULT);
+            // end::create-api-key-execute
+
+            // tag::create-api-key-response
+            SecureString apiKey = createApiKeyResponse.getKey(); // <1>
+            Instant apiKeyExpiration = createApiKeyResponse.getExpiration(); // <2>
+            // end::create-api-key-response
+            assertThat(createApiKeyResponse.getName(), equalTo(name));
+            assertNotNull(apiKey);
+            assertNotNull(apiKeyExpiration);
+        }
+
+        {
+            final String name = randomAlphaOfLength(5);
+            CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest(name, roles, expiration, refreshPolicy);
+
+            ActionListener<CreateApiKeyResponse> listener;
+            // tag::create-api-key-execute-listener
+            listener = new ActionListener<CreateApiKeyResponse>() {
+                @Override
+                public void onResponse(CreateApiKeyResponse createApiKeyResponse) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            };
+            // end::create-api-key-execute-listener
+
+            // Avoid unused variable warning
+            assertNotNull(listener);
+
+            // Replace the empty listener by a blocking listener in test
+            final PlainActionFuture<CreateApiKeyResponse> future = new PlainActionFuture<>();
+            listener = future;
+
+            // tag::create-api-key-execute-async
+            client.security().createApiKeyAsync(createApiKeyRequest, RequestOptions.DEFAULT, listener); // <1>
+            // end::create-api-key-execute-async
+
+            assertNotNull(future.get(30, TimeUnit.SECONDS));
+            assertThat(future.get().getName(), equalTo(name));
+            assertNotNull(future.get().getKey());
+            assertNotNull(future.get().getExpiration());
+        }
+    }
+
+    public void testGetApiKey() throws Exception {
+        RestHighLevelClient client = highLevelClient();
+
+        List<Role> roles = Collections.singletonList(Role.builder().name("r1").clusterPrivileges(ClusterPrivilegeName.ALL)
+                .indicesPrivileges(IndicesPrivileges.builder().indices("ind-x").privileges(IndexPrivilegeName.ALL).build()).build());
+        final TimeValue expiration = TimeValue.timeValueHours(24);
+        final RefreshPolicy refreshPolicy = randomFrom(RefreshPolicy.values());
+        // Create API Keys
+        CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest("k1", roles, expiration, refreshPolicy);
+        CreateApiKeyResponse createApiKeyResponse1 = client.security().createApiKey(createApiKeyRequest, RequestOptions.DEFAULT);
+        assertThat(createApiKeyResponse1.getName(), equalTo("k1"));
+        assertNotNull(createApiKeyResponse1.getKey());
+
+        final ApiKey expectedApiKeyInfo = new ApiKey(createApiKeyResponse1.getName(), createApiKeyResponse1.getId(), Instant.now(),
+                Instant.now().plusMillis(expiration.getMillis()), false, "test_user", "default_file");
+        {
+            // tag::get-api-key-id-request
+            GetApiKeyRequest getApiKeyRequest = GetApiKeyRequest.usingApiKeyId(createApiKeyResponse1.getId());
+            // end::get-api-key-id-request
+
+            // tag::get-api-key-execute
+            GetApiKeyResponse getApiKeyResponse = client.security().getApiKey(getApiKeyRequest, RequestOptions.DEFAULT);
+            // end::get-api-key-execute
+
+            assertThat(getApiKeyResponse.getApiKeyInfos(), is(notNullValue()));
+            assertThat(getApiKeyResponse.getApiKeyInfos().size(), is(1));
+            verifyApiKey(getApiKeyResponse.getApiKeyInfos().get(0), expectedApiKeyInfo);
+        }
+
+        {
+            // tag::get-api-key-name-request
+            GetApiKeyRequest getApiKeyRequest = GetApiKeyRequest.usingApiKeyName(createApiKeyResponse1.getName());
+            // end::get-api-key-name-request
+
+            GetApiKeyResponse getApiKeyResponse = client.security().getApiKey(getApiKeyRequest, RequestOptions.DEFAULT);
+
+            assertThat(getApiKeyResponse.getApiKeyInfos(), is(notNullValue()));
+            assertThat(getApiKeyResponse.getApiKeyInfos().size(), is(1));
+            verifyApiKey(getApiKeyResponse.getApiKeyInfos().get(0), expectedApiKeyInfo);
+        }
+
+        {
+            // tag::get-realm-api-keys-request
+            GetApiKeyRequest getApiKeyRequest = GetApiKeyRequest.usingRealmName("default_file");
+            // end::get-realm-api-keys-request
+
+            GetApiKeyResponse getApiKeyResponse = client.security().getApiKey(getApiKeyRequest, RequestOptions.DEFAULT);
+
+            assertThat(getApiKeyResponse.getApiKeyInfos(), is(notNullValue()));
+            assertThat(getApiKeyResponse.getApiKeyInfos().size(), is(1));
+            verifyApiKey(getApiKeyResponse.getApiKeyInfos().get(0), expectedApiKeyInfo);
+        }
+
+        {
+            // tag::get-user-api-keys-request
+            GetApiKeyRequest getApiKeyRequest = GetApiKeyRequest.usingUserName("test_user");
+            // end::get-user-api-keys-request
+
+            GetApiKeyResponse getApiKeyResponse = client.security().getApiKey(getApiKeyRequest, RequestOptions.DEFAULT);
+
+            assertThat(getApiKeyResponse.getApiKeyInfos(), is(notNullValue()));
+            assertThat(getApiKeyResponse.getApiKeyInfos().size(), is(1));
+            verifyApiKey(getApiKeyResponse.getApiKeyInfos().get(0), expectedApiKeyInfo);
+        }
+
+        {
+            // tag::get-user-realm-api-keys-request
+            GetApiKeyRequest getApiKeyRequest = GetApiKeyRequest.usingRealmAndUserName("default_file", "test_user");
+            // end::get-user-realm-api-keys-request
+
+            // tag::get-api-key-response
+            GetApiKeyResponse getApiKeyResponse = client.security().getApiKey(getApiKeyRequest, RequestOptions.DEFAULT);
+            // end::get-api-key-response
+
+            assertThat(getApiKeyResponse.getApiKeyInfos(), is(notNullValue()));
+            assertThat(getApiKeyResponse.getApiKeyInfos().size(), is(1));
+            verifyApiKey(getApiKeyResponse.getApiKeyInfos().get(0), expectedApiKeyInfo);
+        }
+
+        {
+            GetApiKeyRequest getApiKeyRequest = GetApiKeyRequest.usingApiKeyId(createApiKeyResponse1.getId());
+
+            ActionListener<GetApiKeyResponse> listener;
+            // tag::get-api-key-execute-listener
+            listener = new ActionListener<GetApiKeyResponse>() {
+                @Override
+                public void onResponse(GetApiKeyResponse getApiKeyResponse) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            };
+            // end::get-api-key-execute-listener
+
+            // Avoid unused variable warning
+            assertNotNull(listener);
+
+            // Replace the empty listener by a blocking listener in test
+            final PlainActionFuture<GetApiKeyResponse> future = new PlainActionFuture<>();
+            listener = future;
+
+            // tag::get-api-key-execute-async
+            client.security().getApiKeyAsync(getApiKeyRequest, RequestOptions.DEFAULT, listener); // <1>
+            // end::get-api-key-execute-async
+
+            final GetApiKeyResponse response = future.get(30, TimeUnit.SECONDS);
+            assertNotNull(response);
+
+            assertThat(response.getApiKeyInfos(), is(notNullValue()));
+            assertThat(response.getApiKeyInfos().size(), is(1));
+            verifyApiKey(response.getApiKeyInfos().get(0), expectedApiKeyInfo);
+        }
+    }
+
+    private void verifyApiKey(final ApiKey actual, final ApiKey expected) {
+        assertThat(actual.getId(), is(expected.getId()));
+        assertThat(actual.getName(), is(expected.getName()));
+        assertThat(actual.getUsername(), is(expected.getUsername()));
+        assertThat(actual.getRealm(), is(expected.getRealm()));
+        assertThat(actual.isInvalidated(), is(expected.isInvalidated()));
+        assertThat(actual.getExpiration(), is(greaterThan(Instant.now())));
+    }
+
+    public void testInvalidateApiKey() throws Exception {
+        RestHighLevelClient client = highLevelClient();
+
+        List<Role> roles = Collections.singletonList(Role.builder().name("r1").clusterPrivileges(ClusterPrivilegeName.ALL)
+                .indicesPrivileges(IndicesPrivileges.builder().indices("ind-x").privileges(IndexPrivilegeName.ALL).build()).build());
+        final TimeValue expiration = TimeValue.timeValueHours(24);
+        final RefreshPolicy refreshPolicy = randomFrom(RefreshPolicy.values());
+        // Create API Keys
+        CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest("k1", roles, expiration, refreshPolicy);
+        CreateApiKeyResponse createApiKeyResponse1 = client.security().createApiKey(createApiKeyRequest, RequestOptions.DEFAULT);
+        assertThat(createApiKeyResponse1.getName(), equalTo("k1"));
+        assertNotNull(createApiKeyResponse1.getKey());
+
+        {
+            // tag::invalidate-api-key-id-request
+            InvalidateApiKeyRequest invalidateApiKeyRequest = InvalidateApiKeyRequest.usingApiKeyId(createApiKeyResponse1.getId());
+            // end::invalidate-api-key-id-request
+
+            // tag::invalidate-api-key-execute
+            InvalidateApiKeyResponse invalidateApiKeyResponse = client.security().invalidateApiKey(invalidateApiKeyRequest,
+                    RequestOptions.DEFAULT);
+            // end::invalidate-api-key-execute
+
+            final List<ElasticsearchException> errors = invalidateApiKeyResponse.getErrors();
+            final List<String> invalidatedApiKeyIds = invalidateApiKeyResponse.getInvalidatedApiKeys();
+            final List<String> previouslyInvalidatedApiKeyIds = invalidateApiKeyResponse.getPreviouslyInvalidatedApiKeys();
+
+            assertTrue(errors.isEmpty());
+            List<String> expectedInvalidatedApiKeyIds = Arrays.asList(createApiKeyResponse1.getId());
+            assertThat(invalidatedApiKeyIds, containsInAnyOrder(expectedInvalidatedApiKeyIds.toArray(Strings.EMPTY_ARRAY)));
+            assertThat(previouslyInvalidatedApiKeyIds.size(), equalTo(0));
+        }
+
+        {
+            createApiKeyRequest = new CreateApiKeyRequest("k2", roles, expiration, refreshPolicy);
+            CreateApiKeyResponse createApiKeyResponse2 = client.security().createApiKey(createApiKeyRequest, RequestOptions.DEFAULT);
+            assertThat(createApiKeyResponse2.getName(), equalTo("k2"));
+            assertNotNull(createApiKeyResponse2.getKey());
+
+            // tag::invalidate-api-key-name-request
+            InvalidateApiKeyRequest invalidateApiKeyRequest = InvalidateApiKeyRequest.usingApiKeyName(createApiKeyResponse2.getName());
+            // end::invalidate-api-key-name-request
+
+            InvalidateApiKeyResponse invalidateApiKeyResponse = client.security().invalidateApiKey(invalidateApiKeyRequest,
+                    RequestOptions.DEFAULT);
+
+            final List<ElasticsearchException> errors = invalidateApiKeyResponse.getErrors();
+            final List<String> invalidatedApiKeyIds = invalidateApiKeyResponse.getInvalidatedApiKeys();
+            final List<String> previouslyInvalidatedApiKeyIds = invalidateApiKeyResponse.getPreviouslyInvalidatedApiKeys();
+
+            assertTrue(errors.isEmpty());
+            List<String> expectedInvalidatedApiKeyIds = Arrays.asList(createApiKeyResponse2.getId());
+            assertThat(invalidatedApiKeyIds, containsInAnyOrder(expectedInvalidatedApiKeyIds.toArray(Strings.EMPTY_ARRAY)));
+            assertThat(previouslyInvalidatedApiKeyIds.size(), equalTo(0));
+        }
+
+        {
+            createApiKeyRequest = new CreateApiKeyRequest("k3", roles, expiration, refreshPolicy);
+            CreateApiKeyResponse createApiKeyResponse3 = client.security().createApiKey(createApiKeyRequest, RequestOptions.DEFAULT);
+            assertThat(createApiKeyResponse3.getName(), equalTo("k3"));
+            assertNotNull(createApiKeyResponse3.getKey());
+
+            // tag::invalidate-realm-api-keys-request
+            InvalidateApiKeyRequest invalidateApiKeyRequest = InvalidateApiKeyRequest.usingRealmName("default_file");
+            // end::invalidate-realm-api-keys-request
+
+            InvalidateApiKeyResponse invalidateApiKeyResponse = client.security().invalidateApiKey(invalidateApiKeyRequest,
+                    RequestOptions.DEFAULT);
+
+            final List<ElasticsearchException> errors = invalidateApiKeyResponse.getErrors();
+            final List<String> invalidatedApiKeyIds = invalidateApiKeyResponse.getInvalidatedApiKeys();
+            final List<String> previouslyInvalidatedApiKeyIds = invalidateApiKeyResponse.getPreviouslyInvalidatedApiKeys();
+
+            assertTrue(errors.isEmpty());
+            List<String> expectedInvalidatedApiKeyIds = Arrays.asList(createApiKeyResponse3.getId());
+            assertThat(invalidatedApiKeyIds, containsInAnyOrder(expectedInvalidatedApiKeyIds.toArray(Strings.EMPTY_ARRAY)));
+            assertThat(previouslyInvalidatedApiKeyIds.size(), equalTo(0));
+        }
+
+        {
+            createApiKeyRequest = new CreateApiKeyRequest("k4", roles, expiration, refreshPolicy);
+            CreateApiKeyResponse createApiKeyResponse4 = client.security().createApiKey(createApiKeyRequest, RequestOptions.DEFAULT);
+            assertThat(createApiKeyResponse4.getName(), equalTo("k4"));
+            assertNotNull(createApiKeyResponse4.getKey());
+
+            // tag::invalidate-user-api-keys-request
+            InvalidateApiKeyRequest invalidateApiKeyRequest = InvalidateApiKeyRequest.usingUserName("test_user");
+            // end::invalidate-user-api-keys-request
+
+            InvalidateApiKeyResponse invalidateApiKeyResponse = client.security().invalidateApiKey(invalidateApiKeyRequest,
+                    RequestOptions.DEFAULT);
+
+            final List<ElasticsearchException> errors = invalidateApiKeyResponse.getErrors();
+            final List<String> invalidatedApiKeyIds = invalidateApiKeyResponse.getInvalidatedApiKeys();
+            final List<String> previouslyInvalidatedApiKeyIds = invalidateApiKeyResponse.getPreviouslyInvalidatedApiKeys();
+
+            assertTrue(errors.isEmpty());
+            List<String> expectedInvalidatedApiKeyIds = Arrays.asList(createApiKeyResponse4.getId());
+            assertThat(invalidatedApiKeyIds, containsInAnyOrder(expectedInvalidatedApiKeyIds.toArray(Strings.EMPTY_ARRAY)));
+            assertThat(previouslyInvalidatedApiKeyIds.size(), equalTo(0));
+        }
+
+        {
+            createApiKeyRequest = new CreateApiKeyRequest("k5", roles, expiration, refreshPolicy);
+            CreateApiKeyResponse createApiKeyResponse5 = client.security().createApiKey(createApiKeyRequest, RequestOptions.DEFAULT);
+            assertThat(createApiKeyResponse5.getName(), equalTo("k5"));
+            assertNotNull(createApiKeyResponse5.getKey());
+
+            // tag::invalidate-user-realm-api-keys-request
+            InvalidateApiKeyRequest invalidateApiKeyRequest = InvalidateApiKeyRequest.usingRealmAndUserName("default_file", "test_user");
+            // end::invalidate-user-realm-api-keys-request
+
+            // tag::invalidate-api-key-response
+            InvalidateApiKeyResponse invalidateApiKeyResponse = client.security().invalidateApiKey(invalidateApiKeyRequest,
+                    RequestOptions.DEFAULT);
+            // end::invalidate-api-key-response
+
+            final List<ElasticsearchException> errors = invalidateApiKeyResponse.getErrors();
+            final List<String> invalidatedApiKeyIds = invalidateApiKeyResponse.getInvalidatedApiKeys();
+            final List<String> previouslyInvalidatedApiKeyIds = invalidateApiKeyResponse.getPreviouslyInvalidatedApiKeys();
+
+            assertTrue(errors.isEmpty());
+            List<String> expectedInvalidatedApiKeyIds = Arrays.asList(createApiKeyResponse5.getId());
+            assertThat(invalidatedApiKeyIds, containsInAnyOrder(expectedInvalidatedApiKeyIds.toArray(Strings.EMPTY_ARRAY)));
+            assertThat(previouslyInvalidatedApiKeyIds.size(), equalTo(0));
+        }
+
+        {
+            createApiKeyRequest = new CreateApiKeyRequest("k6", roles, expiration, refreshPolicy);
+            CreateApiKeyResponse createApiKeyResponse6 = client.security().createApiKey(createApiKeyRequest, RequestOptions.DEFAULT);
+            assertThat(createApiKeyResponse6.getName(), equalTo("k6"));
+            assertNotNull(createApiKeyResponse6.getKey());
+
+            InvalidateApiKeyRequest invalidateApiKeyRequest = InvalidateApiKeyRequest.usingApiKeyId(createApiKeyResponse6.getId());
+
+            ActionListener<InvalidateApiKeyResponse> listener;
+            // tag::invalidate-api-key-execute-listener
+            listener = new ActionListener<InvalidateApiKeyResponse>() {
+                @Override
+                public void onResponse(InvalidateApiKeyResponse invalidateApiKeyResponse) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            };
+            // end::invalidate-api-key-execute-listener
+
+            // Avoid unused variable warning
+            assertNotNull(listener);
+
+            // Replace the empty listener by a blocking listener in test
+            final PlainActionFuture<InvalidateApiKeyResponse> future = new PlainActionFuture<>();
+            listener = future;
+
+            // tag::invalidate-api-key-execute-async
+            client.security().invalidateApiKeyAsync(invalidateApiKeyRequest, RequestOptions.DEFAULT, listener); // <1>
+            // end::invalidate-api-key-execute-async
+
+            final InvalidateApiKeyResponse response = future.get(30, TimeUnit.SECONDS);
+            assertNotNull(response);
+            final List<String> invalidatedApiKeyIds = response.getInvalidatedApiKeys();
+            List<String> expectedInvalidatedApiKeyIds = Arrays.asList(createApiKeyResponse6.getId());
+            assertTrue(response.getErrors().isEmpty());
+            assertThat(invalidatedApiKeyIds, containsInAnyOrder(expectedInvalidatedApiKeyIds.toArray(Strings.EMPTY_ARRAY)));
+            assertThat(response.getPreviouslyInvalidatedApiKeys().size(), equalTo(0));
+        }
+    }
 }

client/rest-high-level/src/test/java/org/elasticsearch/client/security/CreateApiKeyRequestTests.java

@@ -0,0 +1,105 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.security;
+
+import org.elasticsearch.client.security.user.privileges.IndicesPrivileges;
+import org.elasticsearch.client.security.user.privileges.Role;
+import org.elasticsearch.client.security.user.privileges.Role.ClusterPrivilegeName;
+import org.elasticsearch.client.security.user.privileges.Role.IndexPrivilegeName;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class CreateApiKeyRequestTests extends ESTestCase {
+
+    public void test() throws IOException {
+        List<Role> roles = new ArrayList<>();
+        roles.add(Role.builder().name("r1").clusterPrivileges(ClusterPrivilegeName.ALL)
+                .indicesPrivileges(IndicesPrivileges.builder().indices("ind-x").privileges(IndexPrivilegeName.ALL).build()).build());
+        roles.add(Role.builder().name("r2").clusterPrivileges(ClusterPrivilegeName.ALL)
+                .indicesPrivileges(IndicesPrivileges.builder().indices("ind-y").privileges(IndexPrivilegeName.ALL).build()).build());
+
+        CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest("api-key", roles, null, null);
+        final XContentBuilder builder = XContentFactory.jsonBuilder();
+        createApiKeyRequest.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        final String output = Strings.toString(builder);
+        assertThat(output, equalTo(
+                "{\"name\":\"api-key\",\"role_descriptors\":{\"r1\":{\"applications\":[],\"cluster\":[\"all\"],\"indices\":[{\"names\":"
+                        + "[\"ind-x\"],\"privileges\":[\"all\"],\"allow_restricted_indices\":false}],\"metadata\":{},\"run_as\":[]},"
+                        + "\"r2\":{\"applications\":[],\"cluster\":"
+                        + "[\"all\"],\"indices\":[{\"names\":[\"ind-y\"],\"privileges\":[\"all\"],\"allow_restricted_indices\":false}],"
+                        + "\"metadata\":{},\"run_as\":[]}}}"));
+    }
+
+    public void testEqualsHashCode() {
+        final String name = randomAlphaOfLength(5);
+        List<Role> roles = Collections.singletonList(Role.builder().name("r1").clusterPrivileges(ClusterPrivilegeName.ALL)
+                .indicesPrivileges(IndicesPrivileges.builder().indices("ind-x").privileges(IndexPrivilegeName.ALL).build()).build());
+        final TimeValue expiration = null;
+        final RefreshPolicy refreshPolicy = randomFrom(RefreshPolicy.values());
+
+        CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest(name, roles, expiration, refreshPolicy);
+
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(createApiKeyRequest, (original) -> {
+            return new CreateApiKeyRequest(original.getName(), original.getRoles(), original.getExpiration(), original.getRefreshPolicy());
+        });
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(createApiKeyRequest, (original) -> {
+            return new CreateApiKeyRequest(original.getName(), original.getRoles(), original.getExpiration(), original.getRefreshPolicy());
+        }, CreateApiKeyRequestTests::mutateTestItem);
+    }
+
+    private static CreateApiKeyRequest mutateTestItem(CreateApiKeyRequest original) {
+        switch (randomIntBetween(0, 3)) {
+        case 0:
+            return new CreateApiKeyRequest(randomAlphaOfLength(5), original.getRoles(), original.getExpiration(),
+                    original.getRefreshPolicy());
+        case 1:
+            return new CreateApiKeyRequest(original.getName(),
+                    Collections.singletonList(Role.builder().name(randomAlphaOfLength(6)).clusterPrivileges(ClusterPrivilegeName.ALL)
+                            .indicesPrivileges(
+                                    IndicesPrivileges.builder().indices(randomAlphaOfLength(4)).privileges(IndexPrivilegeName.ALL).build())
+                            .build()),
+                    original.getExpiration(), original.getRefreshPolicy());
+        case 2:
+            return new CreateApiKeyRequest(original.getName(), original.getRoles(), TimeValue.timeValueSeconds(10000),
+                    original.getRefreshPolicy());
+        case 3:
+            List<RefreshPolicy> values = Arrays.stream(RefreshPolicy.values()).filter(rp -> rp != original.getRefreshPolicy())
+                    .collect(Collectors.toList());
+            return new CreateApiKeyRequest(original.getName(), original.getRoles(), original.getExpiration(), randomFrom(values));
+        default:
+            return new CreateApiKeyRequest(randomAlphaOfLength(5), original.getRoles(), original.getExpiration(),
+                    original.getRefreshPolicy());
+        }
+    }
+}

client/rest-high-level/src/test/java/org/elasticsearch/client/security/CreateApiKeyResponseTests.java

@@ -0,0 +1,101 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.security;
+
+import org.elasticsearch.common.CharArrays;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.Arrays;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class CreateApiKeyResponseTests extends ESTestCase {
+
+    public void testFromXContent() throws IOException {
+        final String id = randomAlphaOfLengthBetween(4, 8);
+        final String name = randomAlphaOfLength(5);
+        final SecureString apiKey = UUIDs.randomBase64UUIDSecureString();
+        final Instant expiration = randomBoolean() ? null : Instant.ofEpochMilli(10000);
+
+        final XContentType xContentType = randomFrom(XContentType.values());
+        final XContentBuilder builder = XContentFactory.contentBuilder(xContentType);
+        builder.startObject().field("id", id).field("name", name);
+        if (expiration != null) {
+            builder.field("expiration", expiration.toEpochMilli());
+        }
+        byte[] charBytes = CharArrays.toUtf8Bytes(apiKey.getChars());
+        try {
+            builder.field("api_key").utf8Value(charBytes, 0, charBytes.length);
+        } finally {
+            Arrays.fill(charBytes, (byte) 0);
+        }
+        builder.endObject();
+        BytesReference xContent = BytesReference.bytes(builder);
+
+        final CreateApiKeyResponse response = CreateApiKeyResponse.fromXContent(createParser(xContentType.xContent(), xContent));
+        assertThat(response.getId(), equalTo(id));
+        assertThat(response.getName(), equalTo(name));
+        assertThat(response.getKey(), equalTo(apiKey));
+        if (expiration != null) {
+            assertThat(response.getExpiration(), equalTo(expiration));
+        }
+    }
+
+    public void testEqualsHashCode() {
+        final String id = randomAlphaOfLengthBetween(4, 8);
+        final String name = randomAlphaOfLength(5);
+        final SecureString apiKey = UUIDs.randomBase64UUIDSecureString();
+        final Instant expiration = Instant.ofEpochMilli(10000);
+        CreateApiKeyResponse createApiKeyResponse = new CreateApiKeyResponse(name, id, apiKey, expiration);
+
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(createApiKeyResponse, (original) -> {
+            return new CreateApiKeyResponse(original.getName(), original.getId(), original.getKey(), original.getExpiration());
+        });
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(createApiKeyResponse, (original) -> {
+            return new CreateApiKeyResponse(original.getName(), original.getId(), original.getKey(), original.getExpiration());
+        }, CreateApiKeyResponseTests::mutateTestItem);
+    }
+
+    private static CreateApiKeyResponse mutateTestItem(CreateApiKeyResponse original) {
+        switch (randomIntBetween(0, 3)) {
+        case 0:
+            return new CreateApiKeyResponse(randomAlphaOfLength(7), original.getId(), original.getKey(), original.getExpiration());
+        case 1:
+            return new CreateApiKeyResponse(original.getName(), randomAlphaOfLengthBetween(4, 8), original.getKey(),
+                    original.getExpiration());
+        case 2:
+            return new CreateApiKeyResponse(original.getName(), original.getId(), UUIDs.randomBase64UUIDSecureString(),
+                    original.getExpiration());
+        case 3:
+            return new CreateApiKeyResponse(original.getName(), original.getId(), original.getKey(), Instant.ofEpochMilli(150000));
+        default:
+            return new CreateApiKeyResponse(randomAlphaOfLength(7), original.getId(), original.getKey(), original.getExpiration());
+        }
+    }
+}

client/rest-high-level/src/test/java/org/elasticsearch/client/security/GetApiKeyRequestTests.java

@@ -0,0 +1,72 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.security;
+
+import org.elasticsearch.client.ValidationException;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Optional;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class GetApiKeyRequestTests extends ESTestCase {
+
+    public void testRequestValidation() {
+        GetApiKeyRequest request = GetApiKeyRequest.usingApiKeyId(randomAlphaOfLength(5));
+        Optional<ValidationException> ve = request.validate();
+        assertFalse(ve.isPresent());
+        request = GetApiKeyRequest.usingApiKeyName(randomAlphaOfLength(5));
+        ve = request.validate();
+        assertFalse(ve.isPresent());
+        request = GetApiKeyRequest.usingRealmName(randomAlphaOfLength(5));
+        ve = request.validate();
+        assertFalse(ve.isPresent());
+        request = GetApiKeyRequest.usingUserName(randomAlphaOfLength(5));
+        ve = request.validate();
+        assertFalse(ve.isPresent());
+        request = GetApiKeyRequest.usingRealmAndUserName(randomAlphaOfLength(5), randomAlphaOfLength(7));
+        ve = request.validate();
+        assertFalse(ve.isPresent());
+    }
+
+    public void testRequestValidationFailureScenarios() throws IOException {
+        String[][] inputs = new String[][] {
+                { randomFrom(new String[] { null, "" }), randomFrom(new String[] { null, "" }), randomFrom(new String[] { null, "" }),
+                        randomFrom(new String[] { null, "" }) },
+                { randomFrom(new String[] { null, "" }), "user", "api-kid", "api-kname" },
+                { "realm", randomFrom(new String[] { null, "" }), "api-kid", "api-kname" },
+                { "realm", "user", "api-kid", randomFrom(new String[] { null, "" }) },
+                { randomFrom(new String[] { null, "" }), randomFrom(new String[] { null, "" }), "api-kid", "api-kname" } };
+        String[] expectedErrorMessages = new String[] { "One of [api key id, api key name, username, realm name] must be specified",
+                "username or realm name must not be specified when the api key id or api key name is specified",
+                "username or realm name must not be specified when the api key id or api key name is specified",
+                "username or realm name must not be specified when the api key id or api key name is specified",
+                "only one of [api key id, api key name] can be specified" };
+
+        for (int i = 0; i < inputs.length; i++) {
+            final int caseNo = i;
+            IllegalArgumentException ve = expectThrows(IllegalArgumentException.class,
+                    () -> new GetApiKeyRequest(inputs[caseNo][0], inputs[caseNo][1], inputs[caseNo][2], inputs[caseNo][3]));
+            assertNotNull(ve);
+            assertThat(ve.getMessage(), equalTo(expectedErrorMessages[caseNo]));
+        }
+    }
+}

client/rest-high-level/src/test/java/org/elasticsearch/client/security/GetApiKeyResponseTests.java

@@ -0,0 +1,100 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.security;
+
+import org.elasticsearch.client.security.support.ApiKey;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.Arrays;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class GetApiKeyResponseTests extends ESTestCase {
+
+    public void testFromXContent() throws IOException {
+        ApiKey apiKeyInfo1 = createApiKeyInfo("name1", "id-1", Instant.ofEpochMilli(100000L), Instant.ofEpochMilli(10000000L), false,
+                "user-a", "realm-x");
+        ApiKey apiKeyInfo2 = createApiKeyInfo("name2", "id-2", Instant.ofEpochMilli(100000L), Instant.ofEpochMilli(10000000L), true,
+                "user-b", "realm-y");
+        GetApiKeyResponse response = new GetApiKeyResponse(Arrays.asList(apiKeyInfo1, apiKeyInfo2));
+        final XContentType xContentType = randomFrom(XContentType.values());
+        final XContentBuilder builder = XContentFactory.contentBuilder(xContentType);
+        toXContent(response, builder);
+        BytesReference xContent = BytesReference.bytes(builder);
+        GetApiKeyResponse responseParsed = GetApiKeyResponse.fromXContent(createParser(xContentType.xContent(), xContent));
+        assertThat(responseParsed, equalTo(response));
+    }
+
+    private void toXContent(GetApiKeyResponse response, final XContentBuilder builder) throws IOException {
+        builder.startObject();
+        builder.startArray("api_keys");
+        for (ApiKey apiKey : response.getApiKeyInfos()) {
+        builder.startObject()
+        .field("id", apiKey.getId())
+        .field("name", apiKey.getName())
+        .field("creation", apiKey.getCreation().toEpochMilli());
+        if (apiKey.getExpiration() != null) {
+            builder.field("expiration", apiKey.getExpiration().toEpochMilli());
+        }
+        builder.field("invalidated", apiKey.isInvalidated())
+        .field("username", apiKey.getUsername())
+        .field("realm", apiKey.getRealm());
+        builder.endObject();
+        }
+        builder.endArray();
+        builder.endObject();
+    }
+
+    public void testEqualsHashCode() {
+        ApiKey apiKeyInfo1 = createApiKeyInfo("name1", "id-1", Instant.ofEpochMilli(100000L), Instant.ofEpochMilli(10000000L), false,
+                "user-a", "realm-x");
+        GetApiKeyResponse response = new GetApiKeyResponse(Arrays.asList(apiKeyInfo1));
+
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(response, (original) -> {
+            return new GetApiKeyResponse(original.getApiKeyInfos());
+        });
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(response, (original) -> {
+            return new GetApiKeyResponse(original.getApiKeyInfos());
+        }, GetApiKeyResponseTests::mutateTestItem);
+    }
+
+    private static GetApiKeyResponse mutateTestItem(GetApiKeyResponse original) {
+        ApiKey apiKeyInfo = createApiKeyInfo("name2", "id-2", Instant.ofEpochMilli(100000L), Instant.ofEpochMilli(10000000L), true,
+                "user-b", "realm-y");
+        switch (randomIntBetween(0, 2)) {
+        case 0:
+            return new GetApiKeyResponse(Arrays.asList(apiKeyInfo));
+        default:
+            return new GetApiKeyResponse(Arrays.asList(apiKeyInfo));
+        }
+    }
+
+    private static ApiKey createApiKeyInfo(String name, String id, Instant creation, Instant expiration, boolean invalidated,
+                                           String username, String realm) {
+        return new ApiKey(name, id, creation, expiration, invalidated, username, realm);
+    }
+}

client/rest-high-level/src/test/java/org/elasticsearch/client/security/InvalidateApiKeyRequestTests.java

@@ -0,0 +1,73 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.security;
+
+import org.elasticsearch.client.ValidationException;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Optional;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+
+public class InvalidateApiKeyRequestTests extends ESTestCase {
+
+    public void testRequestValidation() {
+        InvalidateApiKeyRequest request = InvalidateApiKeyRequest.usingApiKeyId(randomAlphaOfLength(5));
+        Optional<ValidationException> ve = request.validate();
+        assertThat(ve.isPresent(), is(false));
+        request = InvalidateApiKeyRequest.usingApiKeyName(randomAlphaOfLength(5));
+        ve = request.validate();
+        assertThat(ve.isPresent(), is(false));
+        request = InvalidateApiKeyRequest.usingRealmName(randomAlphaOfLength(5));
+        ve = request.validate();
+        assertThat(ve.isPresent(), is(false));
+        request = InvalidateApiKeyRequest.usingUserName(randomAlphaOfLength(5));
+        ve = request.validate();
+        assertThat(ve.isPresent(), is(false));
+        request = InvalidateApiKeyRequest.usingRealmAndUserName(randomAlphaOfLength(5), randomAlphaOfLength(7));
+        ve = request.validate();
+        assertThat(ve.isPresent(), is(false));
+    }
+
+    public void testRequestValidationFailureScenarios() throws IOException {
+        String[][] inputs = new String[][] {
+                { randomFrom(new String[] { null, "" }), randomFrom(new String[] { null, "" }), randomFrom(new String[] { null, "" }),
+                        randomFrom(new String[] { null, "" }) },
+                { randomFrom(new String[] { null, "" }), "user", "api-kid", "api-kname" },
+                { "realm", randomFrom(new String[] { null, "" }), "api-kid", "api-kname" },
+                { "realm", "user", "api-kid", randomFrom(new String[] { null, "" }) },
+                { randomFrom(new String[] { null, "" }), randomFrom(new String[] { null, "" }), "api-kid", "api-kname" } };
+        String[] expectedErrorMessages = new String[] { "One of [api key id, api key name, username, realm name] must be specified",
+                "username or realm name must not be specified when the api key id or api key name is specified",
+                "username or realm name must not be specified when the api key id or api key name is specified",
+                "username or realm name must not be specified when the api key id or api key name is specified",
+                "only one of [api key id, api key name] can be specified" };
+
+        for (int i = 0; i < inputs.length; i++) {
+            final int caseNo = i;
+            IllegalArgumentException ve = expectThrows(IllegalArgumentException.class,
+                    () -> new InvalidateApiKeyRequest(inputs[caseNo][0], inputs[caseNo][1], inputs[caseNo][2], inputs[caseNo][3]));
+            assertNotNull(ve);
+            assertThat(ve.getMessage(), equalTo(expectedErrorMessages[caseNo]));
+        }
+    }
+}

client/rest-high-level/src/test/java/org/elasticsearch/client/security/InvalidateApiKeyResponseTests.java

@@ -0,0 +1,111 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.security;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+
+public class InvalidateApiKeyResponseTests extends ESTestCase {
+
+    public void testFromXContent() throws IOException {
+        List<String> invalidatedApiKeys = Arrays.asList(randomArray(2, 5, String[]::new, () -> randomAlphaOfLength(5)));
+        List<String> previouslyInvalidatedApiKeys = Arrays.asList(randomArray(2, 3, String[]::new, () -> randomAlphaOfLength(5)));
+        List<ElasticsearchException> errors = Arrays.asList(randomArray(2, 5, ElasticsearchException[]::new,
+                () -> new ElasticsearchException(randomAlphaOfLength(5), new IllegalArgumentException(randomAlphaOfLength(4)))));
+
+        final XContentType xContentType = randomFrom(XContentType.values());
+        final XContentBuilder builder = XContentFactory.contentBuilder(xContentType);
+        builder.startObject().array("invalidated_api_keys", invalidatedApiKeys.toArray(Strings.EMPTY_ARRAY))
+                .array("previously_invalidated_api_keys", previouslyInvalidatedApiKeys.toArray(Strings.EMPTY_ARRAY))
+                .field("error_count", errors.size());
+        if (errors.isEmpty() == false) {
+            builder.field("error_details");
+            builder.startArray();
+            for (ElasticsearchException e : errors) {
+                builder.startObject();
+                ElasticsearchException.generateThrowableXContent(builder, ToXContent.EMPTY_PARAMS, e);
+                builder.endObject();
+            }
+            builder.endArray();
+        }
+        builder.endObject();
+        BytesReference xContent = BytesReference.bytes(builder);
+
+        final InvalidateApiKeyResponse response = InvalidateApiKeyResponse.fromXContent(createParser(xContentType.xContent(), xContent));
+        assertThat(response.getInvalidatedApiKeys(), containsInAnyOrder(invalidatedApiKeys.toArray(Strings.EMPTY_ARRAY)));
+        assertThat(response.getPreviouslyInvalidatedApiKeys(),
+                containsInAnyOrder(previouslyInvalidatedApiKeys.toArray(Strings.EMPTY_ARRAY)));
+        assertThat(response.getErrors(), is(notNullValue()));
+        assertThat(response.getErrors().size(), is(errors.size()));
+        assertThat(response.getErrors().get(0).toString(), containsString("type=illegal_argument_exception"));
+        assertThat(response.getErrors().get(1).toString(), containsString("type=illegal_argument_exception"));
+    }
+
+    public void testEqualsHashCode() {
+        List<String> invalidatedApiKeys = Arrays.asList(randomArray(2, 5, String[]::new, () -> randomAlphaOfLength(5)));
+        List<String> previouslyInvalidatedApiKeys = Arrays.asList(randomArray(2, 3, String[]::new, () -> randomAlphaOfLength(5)));
+        List<ElasticsearchException> errors = Arrays.asList(randomArray(2, 5, ElasticsearchException[]::new,
+                () -> new ElasticsearchException(randomAlphaOfLength(5), new IllegalArgumentException(randomAlphaOfLength(4)))));
+        InvalidateApiKeyResponse invalidateApiKeyResponse = new InvalidateApiKeyResponse(invalidatedApiKeys, previouslyInvalidatedApiKeys,
+                errors);
+
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(invalidateApiKeyResponse, (original) -> {
+            return new InvalidateApiKeyResponse(original.getInvalidatedApiKeys(), original.getPreviouslyInvalidatedApiKeys(),
+                    original.getErrors());
+        });
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(invalidateApiKeyResponse, (original) -> {
+            return new InvalidateApiKeyResponse(original.getInvalidatedApiKeys(), original.getPreviouslyInvalidatedApiKeys(),
+                    original.getErrors());
+        }, InvalidateApiKeyResponseTests::mutateTestItem);
+    }
+
+    private static InvalidateApiKeyResponse mutateTestItem(InvalidateApiKeyResponse original) {
+        switch (randomIntBetween(0, 2)) {
+        case 0:
+            return new InvalidateApiKeyResponse(Arrays.asList(randomArray(2, 5, String[]::new, () -> randomAlphaOfLength(5))),
+                    original.getPreviouslyInvalidatedApiKeys(), original.getErrors());
+        case 1:
+            return new InvalidateApiKeyResponse(original.getInvalidatedApiKeys(), Collections.emptyList(), original.getErrors());
+        case 2:
+            return new InvalidateApiKeyResponse(original.getInvalidatedApiKeys(), original.getPreviouslyInvalidatedApiKeys(),
+                    Collections.emptyList());
+        default:
+            return new InvalidateApiKeyResponse(Arrays.asList(randomArray(2, 5, String[]::new, () -> randomAlphaOfLength(5))),
+                    original.getPreviouslyInvalidatedApiKeys(), original.getErrors());
+        }
+    }
+}

docs/java-rest/high-level/security/create-api-key.asciidoc

@@ -0,0 +1,40 @@
+--
+:api: create-api-key
+:request: CreateApiKeyRequest
+:response: CreateApiKeyResponse
+--
+
+[id="{upid}-{api}"]
+=== Create API Key API
+
+API Key can be created using this API.
+
+[id="{upid}-{api}-request"]
+==== Create API Key Request
+
+A +{request}+ contains name for the API key,
+list of role descriptors to define permissions and
+optional expiration for the generated API key.
+If expiration is not provided then by default the API
+keys do not expire.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-request]
+--------------------------------------------------
+
+include::../execution.asciidoc[]
+
+[id="{upid}-{api}-response"]
+==== Create API Key Response
+
+The returned +{response}+ contains an id,
+API key, name for the API key and optional
+expiration.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-response]
+--------------------------------------------------
+<1> the API key that can be used to authenticate to Elasticsearch.
+<2> expiration if the API keys expire

docs/java-rest/high-level/security/get-api-key.asciidoc

@@ -0,0 +1,67 @@
+--
+:api: get-api-key
+:request: GetApiKeyRequest
+:response: GetApiKeyResponse
+--
+
+[id="{upid}-{api}"]
+=== Get API Key information API
+
+API Key(s) information can be retrieved using this API.
+
+[id="{upid}-{api}-request"]
+==== Get API Key Request
+The +{request}+ supports retrieving API key information for
+
+. A specific API key
+
+. All API keys for a specific realm
+
+. All API keys for a specific user
+
+. All API keys for a specific user in a specific realm
+
+===== Retrieve a specific API key by its id
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[get-api-key-id-request]
+--------------------------------------------------
+
+===== Retrieve a specific API key by its name
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[get-api-key-name-request]
+--------------------------------------------------
+
+===== Retrieve all API keys for given realm
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[get-realm-api-keys-request]
+--------------------------------------------------
+
+===== Retrieve all API keys for a given user
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[get-user-api-keys-request]
+--------------------------------------------------
+
+===== Retrieve all API keys for given user in a realm
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[get-user-realm-api-keys-request]
+--------------------------------------------------
+
+include::../execution.asciidoc[]
+
+[id="{upid}-{api}-response"]
+==== Get API Key information API Response
+
+The returned +{response}+ contains the information regarding the API keys that were
+requested.
+
+`api_keys`:: Available using `getApiKeyInfos`, contains list of API keys that were retrieved for this request.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-response]
+--------------------------------------------------

docs/java-rest/high-level/security/invalidate-api-key.asciidoc

@@ -0,0 +1,75 @@
+--
+:api: invalidate-api-key
+:request: InvalidateApiKeyRequest
+:response: InvalidateApiKeyResponse
+--
+
+[id="{upid}-{api}"]
+=== Invalidate API Key API
+
+API Key(s) can be invalidated using this API.
+
+[id="{upid}-{api}-request"]
+==== Invalidate API Key Request
+The +{request}+ supports invalidating
+
+. A specific API key
+
+. All API keys for a specific realm
+
+. All API keys for a specific user
+
+. All API keys for a specific user in a specific realm
+
+===== Specific API key by API key id
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[invalidate-api-key-id-request]
+--------------------------------------------------
+
+===== Specific API key by API key name
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[invalidate-api-key-name-request]
+--------------------------------------------------
+
+===== All API keys for realm
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[invalidate-realm-api-keys-request]
+--------------------------------------------------
+
+===== All API keys for user
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[invalidate-user-api-keys-request]
+--------------------------------------------------
+
+===== All API key for user in realm
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[invalidate-user-realm-api-keys-request]
+--------------------------------------------------
+
+include::../execution.asciidoc[]
+
+[id="{upid}-{api}-response"]
+==== Invalidate API Key Response
+
+The returned +{response}+ contains the information regarding the API keys that the request
+invalidated.
+
+`invalidatedApiKeys`:: Available using `getInvalidatedApiKeys` lists the API keys
+                      that this request invalidated.
+
+`previouslyInvalidatedApiKeys`:: Available using `getPreviouslyInvalidatedApiKeys` lists the API keys
+                                that this request attempted to invalidate
+                                but were already invalid.
+
+`errors`:: Available using `getErrors` contains possible errors that were encountered while
+           attempting to invalidate API keys.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-response]
+--------------------------------------------------

docs/java-rest/high-level/supported-apis.asciidoc

@@ -411,6 +411,9 @@ The Java High Level REST Client supports the following Security APIs:
 * <<{upid}-get-privileges>>
 * <<{upid}-put-privileges>>
 * <<{upid}-delete-privileges>>
+* <<{upid}-create-api-key>>
+* <<{upid}-get-api-key>>
+* <<{upid}-invalidate-api-key>>
 
 include::security/put-user.asciidoc[]
 include::security/get-users.asciidoc[]
@@ -435,6 +438,9 @@ include::security/delete-role-mapping.asciidoc[]
 include::security/create-token.asciidoc[]
 include::security/invalidate-token.asciidoc[]
 include::security/put-privileges.asciidoc[]
+include::security/create-api-key.asciidoc[]
+include::security/get-api-key.asciidoc[]
+include::security/invalidate-api-key.asciidoc[]
 
 == Watcher APIs
 

rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc

@@ -280,6 +280,31 @@ example above), but the same goes for actual values:
 
 The stash should be reset at the beginning of each test file.
 
+=== `transform_and_set`
+
+For some tests, it is necessary to extract a value and transform it from the previous `response`, in
+order to reuse it in a subsequent `do` and other tests.
+Currently, it only has support for `base64EncodeCredentials`, for unknown transformations it will not
+do anything and stash the value as is.
+For instance, when testing you may want to base64 encode username and password for
+`Basic` authorization header:
+
+....
+    - do:
+        index:
+            index: test
+            type:  test
+    - transform_and_set:  { login_creds: "#base64EncodeCredentials(user,password)" }   # stash the base64 encoded credentials of `response.user` and `response.password` as `login_creds`
+    - do:
+        headers:
+            Authorization: Basic ${login_creds} # replace `$login_creds` with the stashed value
+        get:
+            index: test
+            type:  test
+....
+
+Stashed values can be used as described in the `set` section
+
 === `is_true`
 
 The specified key exists and has a true value (ie not `0`, `false`, `undefined`, `null`

server/src/main/java/org/elasticsearch/common/RandomBasedUUIDGenerator.java

@@ -20,6 +20,9 @@
 package org.elasticsearch.common;
 
 
+import org.elasticsearch.common.settings.SecureString;
+
+import java.util.Arrays;
 import java.util.Base64;
 import java.util.Random;
 
@@ -34,12 +37,37 @@ class RandomBasedUUIDGenerator implements UUIDGenerator {
         return getBase64UUID(SecureRandomHolder.INSTANCE);
     }
 
+    /**
+     * Returns a Base64 encoded {@link SecureString} of a Version 4.0 compatible UUID
+     * as defined here: http://www.ietf.org/rfc/rfc4122.txt
+     */
+    public SecureString getBase64UUIDSecureString() {
+        byte[] uuidBytes = null;
+        byte[] encodedBytes = null;
+        try {
+            uuidBytes = getUUIDBytes(SecureRandomHolder.INSTANCE);
+            encodedBytes = Base64.getUrlEncoder().withoutPadding().encode(uuidBytes);
+            return new SecureString(CharArrays.utf8BytesToChars(encodedBytes));
+        } finally {
+            if (uuidBytes != null) {
+                Arrays.fill(uuidBytes, (byte) 0);
+            }
+            if (encodedBytes != null) {
+                Arrays.fill(encodedBytes, (byte) 0);
+            }
+        }
+    }
+
     /**
      * Returns a Base64 encoded version of a Version 4.0 compatible UUID
      * randomly initialized by the given {@link java.util.Random} instance
      * as defined here: http://www.ietf.org/rfc/rfc4122.txt
      */
     public String getBase64UUID(Random random) {
+        return Base64.getUrlEncoder().withoutPadding().encodeToString(getUUIDBytes(random));
+    }
+
+    private byte[] getUUIDBytes(Random random) {
         final byte[] randomBytes = new byte[16];
         random.nextBytes(randomBytes);
         /* Set the version to version 4 (see http://www.ietf.org/rfc/rfc4122.txt)
@@ -48,12 +76,12 @@ class RandomBasedUUIDGenerator implements UUIDGenerator {
          * stamp (bits 4 through 7 of the time_hi_and_version field).*/
         randomBytes[6] &= 0x0f;  /* clear the 4 most significant bits for the version  */
         randomBytes[6] |= 0x40;  /* set the version to 0100 / 0x40 */
-        
-        /* Set the variant: 
+
+        /* Set the variant:
          * The high field of th clock sequence multiplexed with the variant.
          * We set only the MSB of the variant*/
         randomBytes[8] &= 0x3f;  /* clear the 2 most significant bits */
         randomBytes[8] |= 0x80;  /* set the variant (MSB is set)*/
-        return Base64.getUrlEncoder().withoutPadding().encodeToString(randomBytes);
+        return randomBytes;
     }
 }

server/src/main/java/org/elasticsearch/common/UUIDs.java

@@ -19,6 +19,8 @@
 
 package org.elasticsearch.common;
 
+import org.elasticsearch.common.settings.SecureString;
+
 import java.util.Random;
 
 public class UUIDs {
@@ -50,4 +52,9 @@ public class UUIDs {
         return RANDOM_UUID_GENERATOR.getBase64UUID();
     }
 
+    /** Returns a Base64 encoded {@link SecureString} of a Version 4.0 compatible UUID as defined here: http://www.ietf.org/rfc/rfc4122.txt,
+     *  using a private {@code SecureRandom} instance */
+    public static SecureString randomBase64UUIDSecureString() {
+        return RANDOM_UUID_GENERATOR.getBase64UUIDSecureString();
+    }
 }

server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java

@@ -588,6 +588,23 @@ public abstract class StreamInput extends InputStream {
         }
     }
 
+    /**
+     * Read an {@link Instant} from the stream with nanosecond resolution
+     */
+    public final Instant readInstant() throws IOException {
+        return Instant.ofEpochSecond(readLong(), readInt());
+    }
+
+    /**
+     * Read an optional {@link Instant} from the stream. Returns <code>null</code> when
+     * no instant is present.
+     */
+    @Nullable
+    public final Instant readOptionalInstant() throws IOException {
+        final boolean present = readBoolean();
+        return present ? readInstant() : null;
+    }
+
     @SuppressWarnings("unchecked")
     private List readArrayList() throws IOException {
         int size = readArraySize();

server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java

@@ -56,6 +56,7 @@ import java.nio.file.FileSystemLoopException;
 import java.nio.file.NoSuchFileException;
 import java.nio.file.NotDirectoryException;
 import java.time.ZoneId;
+import java.time.Instant;
 import java.time.ZonedDateTime;
 import java.util.Collection;
 import java.util.Collections;
@@ -573,6 +574,26 @@ public abstract class StreamOutput extends OutputStream {
         }
     }
 
+    /**
+     * Writes an {@link Instant} to the stream with nanosecond resolution
+     */
+    public final void writeInstant(Instant instant) throws IOException {
+        writeLong(instant.getEpochSecond());
+        writeInt(instant.getNano());
+    }
+
+    /**
+     * Writes an {@link Instant} to the stream, which could possibly be null
+     */
+    public final void writeOptionalInstant(@Nullable Instant instant) throws IOException {
+        if (instant == null) {
+            writeBoolean(false);
+        } else {
+            writeBoolean(true);
+            writeInstant(instant);
+        }
+    }
+
     private static final Map<Class<?>, Writer> WRITERS;
 
     static {

server/src/main/java/org/elasticsearch/common/util/set/Sets.java

@@ -144,4 +144,19 @@ public final class Sets {
         union.addAll(right);
         return union;
     }
+
+    public static <T> Set<T> intersection(Set<T> set1, Set<T> set2) {
+        Objects.requireNonNull(set1);
+        Objects.requireNonNull(set2);
+        final Set<T> left;
+        final Set<T> right;
+        if (set1.size() < set2.size()) {
+            left = set1;
+            right = set2;
+        } else {
+            left = set2;
+            right = set1;
+        }
+        return left.stream().filter(o -> right.contains(o)).collect(Collectors.toSet());
+    }
 }

server/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java

@@ -30,6 +30,7 @@ import org.elasticsearch.test.ESTestCase;
 import java.io.ByteArrayInputStream;
 import java.io.EOFException;
 import java.io.IOException;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -336,6 +337,37 @@ public class StreamTests extends ESTestCase {
         assertThat(targetSet, equalTo(sourceSet));
     }
 
+    public void testInstantSerialization() throws IOException {
+        final Instant instant = Instant.now();
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            out.writeInstant(instant);
+            try (StreamInput in = out.bytes().streamInput()) {
+                final Instant serialized = in.readInstant();
+                assertEquals(instant, serialized);
+            }
+        }
+    }
+
+    public void testOptionalInstantSerialization() throws IOException {
+        final Instant instant = Instant.now();
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            out.writeOptionalInstant(instant);
+            try (StreamInput in = out.bytes().streamInput()) {
+                final Instant serialized = in.readOptionalInstant();
+                assertEquals(instant, serialized);
+            }
+        }
+
+        final Instant missing = null;
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            out.writeOptionalInstant(missing);
+            try (StreamInput in = out.bytes().streamInput()) {
+                final Instant serialized = in.readOptionalInstant();
+                assertEquals(missing, serialized);
+            }
+        }
+    }
+
     static final class WriteableString implements Writeable {
         final String string;
 

server/src/test/java/org/elasticsearch/common/util/set/SetsTests.java

@@ -28,6 +28,7 @@ import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
+import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 
@@ -56,6 +57,17 @@ public class SetsTests extends ESTestCase {
         }
     }
 
+    public void testIntersection() {
+        final int endExclusive = randomIntBetween(0, 256);
+        final Tuple<Set<Integer>, Set<Integer>> sets = randomSets(endExclusive);
+        final Set<Integer> intersection = Sets.intersection(sets.v1(), sets.v2());
+        final Set<Integer> expectedIntersection = IntStream.range(0, endExclusive)
+                .boxed()
+                .filter(i -> (sets.v1().contains(i) && sets.v2().contains(i)))
+                .collect(Collectors.toSet());
+        assertThat(intersection, containsInAnyOrder(expectedIntersection.toArray(new Integer[0])));
+    }
+
     /**
      * Assert the difference between two sets is as expected.
      *

test/framework/src/main/java/org/elasticsearch/test/rest/yaml/Features.java

@@ -46,7 +46,8 @@ public final class Features {
             "stash_path_replace",
             "warnings",
             "yaml",
-            "contains"
+            "contains",
+            "transform_and_set"
     ));
 
     private Features() {

test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ExecutableSection.java

@@ -40,6 +40,7 @@ public interface ExecutableSection {
     List<NamedXContentRegistry.Entry> DEFAULT_EXECUTABLE_CONTEXTS = unmodifiableList(Arrays.asList(
             new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("do"), DoSection::parse),
             new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("set"), SetSection::parse),
+            new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("transform_and_set"), TransformAndSetSection::parse),
             new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("match"), MatchAssertion::parse),
             new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("is_true"), IsTrueAssertion::parse),
             new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("is_false"), IsFalseAssertion::parse),

test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/TransformAndSetSection.java

@@ -0,0 +1,106 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.rest.yaml.section;
+
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Represents a transform_and_set section:
+ * <p>
+ *
+ * In the following example,<br>
+ * - transform_and_set: { login_creds: "#base64EncodeCredentials(user,password)" }<br>
+ * user and password are from the response which are joined by ':' and Base64 encoded and then stashed as 'login_creds'
+ *
+ */
+public class TransformAndSetSection implements ExecutableSection {
+    public static TransformAndSetSection parse(XContentParser parser) throws IOException {
+        String currentFieldName = null;
+        XContentParser.Token token;
+
+        TransformAndSetSection transformAndStashSection = new TransformAndSetSection(parser.getTokenLocation());
+
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token.isValue()) {
+                transformAndStashSection.addSet(currentFieldName, parser.text());
+            }
+        }
+
+        parser.nextToken();
+
+        if (transformAndStashSection.getStash().isEmpty()) {
+            throw new ParsingException(transformAndStashSection.location, "transform_and_set section must set at least a value");
+        }
+
+        return transformAndStashSection;
+    }
+
+    private final Map<String, String> transformStash = new HashMap<>();
+    private final XContentLocation location;
+
+    public TransformAndSetSection(XContentLocation location) {
+        this.location = location;
+    }
+
+    public void addSet(String stashedField, String transformThis) {
+        transformStash.put(stashedField, transformThis);
+    }
+
+    public Map<String, String> getStash() {
+        return transformStash;
+    }
+
+    @Override
+    public XContentLocation getLocation() {
+        return location;
+    }
+
+    @Override
+    public void execute(ClientYamlTestExecutionContext executionContext) throws IOException {
+        for (Map.Entry<String, String> entry : transformStash.entrySet()) {
+            String key = entry.getKey();
+            String value = entry.getValue();
+            if (value.startsWith("#base64EncodeCredentials(") && value.endsWith(")")) {
+                value = entry.getValue().substring("#base64EncodeCredentials(".length(), entry.getValue().lastIndexOf(")"));
+                String[] idAndPassword = value.split(",");
+                if (idAndPassword.length == 2) {
+                    String credentials = executionContext.response(idAndPassword[0].trim()) + ":"
+                            + executionContext.response(idAndPassword[1].trim());
+                    value = Base64.getEncoder().encodeToString(credentials.getBytes(StandardCharsets.UTF_8));
+                } else {
+                    throw new IllegalArgumentException("base64EncodeCredentials requires a username/id and a password parameters");
+                }
+            }
+            executionContext.stash().stashValue(key, value);
+        }
+    }
+
+}

test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/TransformAndSetSectionTests.java

@@ -0,0 +1,96 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.rest.yaml.section;
+
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.xcontent.yaml.YamlXContent;
+import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext;
+import org.elasticsearch.test.rest.yaml.Stash;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
+
+public class TransformAndSetSectionTests extends AbstractClientYamlTestFragmentParserTestCase {
+
+    public void testParseSingleValue() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                        "{ key: value }"
+        );
+
+        TransformAndSetSection transformAndSet = TransformAndSetSection.parse(parser);
+        assertThat(transformAndSet, notNullValue());
+        assertThat(transformAndSet.getStash(), notNullValue());
+        assertThat(transformAndSet.getStash().size(), equalTo(1));
+        assertThat(transformAndSet.getStash().get("key"), equalTo("value"));
+    }
+
+    public void testParseMultipleValues() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                        "{ key1: value1, key2: value2 }"
+        );
+
+        TransformAndSetSection transformAndSet = TransformAndSetSection.parse(parser);
+        assertThat(transformAndSet, notNullValue());
+        assertThat(transformAndSet.getStash(), notNullValue());
+        assertThat(transformAndSet.getStash().size(), equalTo(2));
+        assertThat(transformAndSet.getStash().get("key1"), equalTo("value1"));
+        assertThat(transformAndSet.getStash().get("key2"), equalTo("value2"));
+    }
+
+    public void testTransformation() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent, "{ login_creds: \"#base64EncodeCredentials(id,api_key)\" }");
+
+        TransformAndSetSection transformAndSet = TransformAndSetSection.parse(parser);
+        assertThat(transformAndSet, notNullValue());
+        assertThat(transformAndSet.getStash(), notNullValue());
+        assertThat(transformAndSet.getStash().size(), equalTo(1));
+        assertThat(transformAndSet.getStash().get("login_creds"), equalTo("#base64EncodeCredentials(id,api_key)"));
+
+        ClientYamlTestExecutionContext executionContext = mock(ClientYamlTestExecutionContext.class);
+        when(executionContext.response("id")).thenReturn("user");
+        when(executionContext.response("api_key")).thenReturn("password");
+        Stash stash = new Stash();
+        when(executionContext.stash()).thenReturn(stash);
+        transformAndSet.execute(executionContext);
+        verify(executionContext).response("id");
+        verify(executionContext).response("api_key");
+        verify(executionContext).stash();
+        assertThat(stash.getValue("$login_creds"),
+                equalTo(Base64.getEncoder().encodeToString("user:password".getBytes(StandardCharsets.UTF_8))));
+        verifyNoMoreInteractions(executionContext);
+    }
+
+    public void testParseSetSectionNoValues() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "{ }"
+        );
+
+        Exception e = expectThrows(ParsingException.class, () -> TransformAndSetSection.parse(parser));
+        assertThat(e.getMessage(), is("transform_and_set section must set at least a value"));
+    }
+}

x-pack/docs/build.gradle

@@ -73,6 +73,7 @@ project.copyRestSpec.from(xpackResources) {
 }
 integTestCluster {
     setting 'xpack.security.enabled', 'true'
+    setting 'xpack.security.authc.api_key.enabled', 'true'
     setting 'xpack.security.authc.token.enabled', 'true'
     // Disable monitoring exporters for the docs tests
     setting 'xpack.monitoring.exporters._local.type', 'local'

x-pack/docs/en/rest-api/security.asciidoc

@@ -51,6 +51,17 @@ without requiring basic authentication:
 * <<security-api-get-token,Get token>>
 * <<security-api-invalidate-token,Invalidate token>>
 
+[float]
+[[security-api-keys]]
+=== API Keys
+
+You can use the following APIs to create, retrieve and invalidate API keys for access
+without requiring basic authentication:
+
+* <<security-api-create-api-key,Create API Key>>
+* <<security-api-get-api-key,Get API Key>>
+* <<security-api-invalidate-api-key,Invalidate API Key>>
+
 [float]
 [[security-user-apis]]
 === Users
@@ -88,3 +99,6 @@ include::security/get-users.asciidoc[]
 include::security/has-privileges.asciidoc[]
 include::security/invalidate-tokens.asciidoc[]
 include::security/ssl.asciidoc[]
+include::security/create-api-keys.asciidoc[]
+include::security/invalidate-api-keys.asciidoc[]
+include::security/get-api-keys.asciidoc[]

x-pack/docs/en/rest-api/security/create-api-keys.asciidoc

@@ -0,0 +1,99 @@
+[role="xpack"]
+[[security-api-create-api-key]]
+=== Create API Key API
+
+Creates an API key for access without requiring basic authentication.
+
+==== Request
+
+`POST /_security/api_key`
+`PUT /_security/api_key`
+
+==== Description
+
+The API keys are created by the {es} API key service, which is automatically enabled
+when you configure TLS on the HTTP interface. See <<tls-http>>. Alternatively,
+you can explicitly enable the `xpack.security.authc.api_key.enabled` setting. When 
+you are running in production mode, a bootstrap check prevents you from enabling 
+the API key service unless you also enable TLS on the HTTP interface. 
+
+A successful create API key API call returns a JSON structure that contains 
+the unique id, the name to identify API key, the API key and the expiration if 
+applicable for the API key in milliseconds. 
+
+NOTE: By default API keys never expire. You can specify expiration at the time of 
+creation for the API keys. 
+
+==== Request Body
+
+The following parameters can be specified in the body of a POST or PUT request:
+
+`name`::
+(string) Specifies the name for this API key.
+
+`role_descriptors`::
+(array-of-role-descriptor) Optional array of role descriptor for this API key. The role descriptor 
+must be a subset of permissions of the authenticated user. The structure of role 
+descriptor is same as the request for create role API. For more details on role 
+see <<security-api-roles, Role Management APIs>>.
+If the role descriptors are not provided then permissions of the authenticated user are applied.
+
+`expiration`::
+(string) Optional expiration time for the API key. By default API keys never expire.
+
+==== Examples
+
+The following example creates an API key:
+
+[source, js]
+------------------------------------------------------------
+POST /_security/api_key
+{
+  "name": "my-api-key",
+  "expiration": "1d", <1>
+  "role_descriptors": { <2>
+    "role-a": {
+      "cluster": ["all"],
+      "index": [
+        {
+          "names": ["index-a*"],
+          "privileges": ["read"]
+        }
+      ]
+    },
+    "role-b": {
+      "cluster": ["all"],
+      "index": [
+        {
+          "names": ["index-b*"],
+          "privileges": ["all"]
+        }
+      ]
+    }
+  }
+}
+------------------------------------------------------------
+// CONSOLE
+<1> optional expiration for the API key being generated. If expiration is not
+ provided then the API keys do not expire.
+<2> optional role descriptors for this API key, if not provided then permissions
+ of authenticated user are applied.
+
+A successful call returns a JSON structure that provides
+API key information.
+
+[source,js]
+--------------------------------------------------
+{
+  "id":"VuaCfGcBCdbkQm-e5aOx", <1>
+  "name":"my-api-key",
+  "expiration":1544068612110, <2>
+  "api_key":"ui2lp2axTNmsyakw9tvNnw" <3>
+}
+--------------------------------------------------
+// TESTRESPONSE[s/VuaCfGcBCdbkQm-e5aOx/$body.id/]
+// TESTRESPONSE[s/1544068612110/$body.expiration/]
+// TESTRESPONSE[s/ui2lp2axTNmsyakw9tvNnw/$body.api_key/]
+<1> unique id for this API key
+<2> optional expiration in milliseconds for this API key
+<3> generated API key

x-pack/docs/en/rest-api/security/get-api-keys.asciidoc

@@ -0,0 +1,118 @@
+[role="xpack"]
+[[security-api-get-api-key]]
+=== Get API Key information API
+++++
+<titleabbrev>Get API key information</titleabbrev>
+++++
+
+Retrieves information for one or more API keys.
+
+==== Request
+
+`GET /_security/api_key`
+
+==== Description
+
+The information for the API keys created by <<security-api-create-api-key,create API Key>> can be retrieved
+using this API.
+
+==== Request Body
+
+The following parameters can be specified in the query parameters of a GET request and
+pertain to retrieving api keys:
+
+`id` (optional)::
+(string) An API key id. This parameter cannot be used with any of `name`, `realm_name` or
+         `username` are used.
+
+`name` (optional)::
+(string) An API key name. This parameter cannot be used with any of `id`, `realm_name` or
+                          `username` are used.
+
+`realm_name` (optional)::
+(string) The name of an authentication realm. This parameter cannot be used with either `id` or `name`.
+
+`username` (optional)::
+(string) The username of a user. This parameter cannot be used with either `id` or `name`.
+
+NOTE: While all parameters are optional, at least one of them is required.
+
+==== Examples
+
+The following example to retrieve the API key identified by specified `id`:
+
+[source,js]
+--------------------------------------------------
+GET /_security/api_key?id=dGhpcyBpcyBub3QgYSByZWFsIHRva2VuIGJ1dCBpdCBpcyBvbmx5IHRlc3QgZGF0YS4gZG8gbm90IHRyeSB0byByZWFkIHRva2VuIQ==
+--------------------------------------------------
+// NOTCONSOLE
+
+whereas the following example to retrieve the API key identified by specified `name`:
+
+[source,js]
+--------------------------------------------------
+GET /_security/api_key?name=hadoop_myuser_key
+--------------------------------------------------
+// NOTCONSOLE
+
+The following example retrieves all API keys for the `native1` realm:
+
+[source,js]
+--------------------------------------------------
+GET /_xpack/api_key?realm_name=native1
+--------------------------------------------------
+// NOTCONSOLE
+
+The following example retrieves all API keys for the user `myuser` in all realms:
+
+[source,js]
+--------------------------------------------------
+GET /_xpack/api_key?username=myuser
+--------------------------------------------------
+// NOTCONSOLE
+
+Finally, the following example retrieves all API keys for the user `myuser` in
+ the `native1` realm immediately:
+
+[source,js]
+--------------------------------------------------
+GET /_xpack/api_key?username=myuser&realm_name=native1
+--------------------------------------------------
+// NOTCONSOLE
+
+A successful call returns a JSON structure that contains the information of one or more API keys that were retrieved.
+
+[source,js]
+--------------------------------------------------
+{
+  "api_keys": [ <1>
+    {
+      "id": "dGhpcyBpcyBub3QgYSByZWFsIHRva2VuIGJ1dCBpdCBpcyBvbmx5IHRlc3QgZGF0YS4gZG8gbm90IHRyeSB0byByZWFkIHRva2VuIQ==", <2>
+      "name": "hadoop_myuser_key", <3>
+      "creation": 1548550550158, <4>
+      "expiration": 1548551550158, <5>
+      "invalidated": false, <6>
+      "username": "myuser", <7>
+      "realm": "native1" <8>
+    },
+    {
+      "id": "api-key-id-2",
+      "name": "api-key-name-2",
+      "creation": 1548550550158,
+      "invalidated": false,
+      "username": "user-y",
+      "realm": "realm-2"
+    }
+  ]
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+<1> The list of API keys that were retrieved for this request.
+<2> Id for the API key
+<3> Name of the API key
+<4> Creation time for the API key in milliseconds
+<5> optional expiration time for the API key in milliseconds
+<6> invalidation status for the API key, `true` if the key has been invalidated else `false`
+<7> principal for which this API key was created
+<8> realm name of the principal for which this API key was created

x-pack/docs/en/rest-api/security/invalidate-api-keys.asciidoc

@@ -0,0 +1,140 @@
+[role="xpack"]
+[[security-api-invalidate-api-key]]
+=== Invalidate API Key API
+++++
+<titleabbrev>Invalidate API key</titleabbrev>
+++++
+
+Invalidates one or more API keys.
+
+==== Request
+
+`DELETE /_security/api_key`
+
+==== Description
+
+The API keys created by <<security-api-create-api-key,create API Key>> can be invalidated
+using this API.
+
+==== Request Body
+
+The following parameters can be specified in the body of a DELETE request and
+pertain to invalidating api keys:
+
+`id` (optional)::
+(string) An API key id. This parameter cannot be used with any of `name`, `realm_name` or
+         `username` are used.
+
+`name` (optional)::
+(string) An API key name. This parameter cannot be used with any of `id`, `realm_name` or
+                          `username` are used.
+
+`realm_name` (optional)::
+(string) The name of an authentication realm. This parameter cannot be used with either `api_key_id` or `api_key_name`.
+
+`username` (optional)::
+(string) The username of a user. This parameter cannot be used with either `api_key_id` or `api_key_name`.
+
+NOTE: While all parameters are optional, at least one of them is required.
+
+==== Examples
+
+The following example invalidates the API key identified by specified `id` immediately:
+
+[source,js]
+--------------------------------------------------
+DELETE /_security/api_key
+{
+  "id" : "dGhpcyBpcyBub3QgYSByZWFsIHRva2VuIGJ1dCBpdCBpcyBvbmx5IHRlc3QgZGF0YS4gZG8gbm90IHRyeSB0byByZWFkIHRva2VuIQ=="
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+whereas the following example invalidates the API key identified by specified `name` immediately:
+
+[source,js]
+--------------------------------------------------
+DELETE /_security/api_key
+{
+  "name" : "hadoop_myuser_key"
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+The following example invalidates all API keys for the `native1` realm immediately:
+
+[source,js]
+--------------------------------------------------
+DELETE /_xpack/api_key
+{
+  "realm_name" : "native1"
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+The following example invalidates all API keys for the user `myuser` in all realms immediately:
+
+[source,js]
+--------------------------------------------------
+DELETE /_xpack/api_key
+{
+  "username" : "myuser"
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+Finally, the following example invalidates all API keys for the user `myuser` in
+ the `native1` realm immediately:
+
+[source,js]
+--------------------------------------------------
+DELETE /_xpack/api_key
+{
+  "username" : "myuser",
+  "realm_name" : "native1"
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+A successful call returns a JSON structure that contains the ids of the API keys that were invalidated, the ids
+of the API keys that had already been invalidated, and potentially a list of errors encountered while invalidating
+specific api keys.
+
+[source,js]
+--------------------------------------------------
+{
+  "invalidated_api_keys": [ <1>
+    "api-key-id-1"
+  ],
+  "previously_invalidated_api_keys": [ <2>
+    "api-key-id-2",
+    "api-key-id-3"
+  ],
+  "error_count": 2, <3>
+  "error_details": [ <4>
+    {
+      "type": "exception",
+      "reason": "error occurred while invalidating api keys",
+      "caused_by": {
+        "type": "illegal_argument_exception",
+        "reason": "invalid api key id"
+      }
+    },
+    {
+      "type": "exception",
+      "reason": "error occurred while invalidating api keys",
+      "caused_by": {
+        "type": "illegal_argument_exception",
+        "reason": "invalid api key id"
+      }
+    }
+  ]
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+<1> The ids of the API keys that were invalidated as part of this request.
+<2> The ids of the API keys that were already invalidated.
+<3> The number of errors that were encountered when invalidating the API keys.
+<4> Details about these errors. This field is not present in the response when
+    `error_count` is 0.

x-pack/plugin/build.gradle

@@ -133,6 +133,7 @@ integTestCluster {
   setting 'xpack.monitoring.exporters._local.type', 'local'
   setting 'xpack.monitoring.exporters._local.enabled', 'false'
   setting 'xpack.security.authc.token.enabled', 'true'
+  setting 'xpack.security.authc.api_key.enabled', 'true'
   setting 'xpack.security.transport.ssl.enabled', 'true'
   setting 'xpack.security.transport.ssl.key', nodeKey.name
   setting 'xpack.security.transport.ssl.certificate', nodeCert.name

x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrLicenseChecker.java

@@ -13,21 +13,21 @@ import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
-import org.elasticsearch.action.support.ContextPreservingActionListener;
 import org.elasticsearch.action.admin.indices.stats.IndexShardStats;
 import org.elasticsearch.action.admin.indices.stats.IndexStats;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.admin.indices.stats.ShardStats;
+import org.elasticsearch.action.support.ContextPreservingActionListener;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.FilterClient;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.CheckedConsumer;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.engine.CommitStats;
 import org.elasticsearch.index.engine.Engine;
@@ -35,14 +35,15 @@ import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.license.RemoteClusterLicenseChecker;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.xpack.ccr.action.ShardFollowTask;
 import org.elasticsearch.xpack.ccr.action.ShardChangesAction;
+import org.elasticsearch.xpack.ccr.action.ShardFollowTask;
 import org.elasticsearch.xpack.core.XPackPlugin;
 import org.elasticsearch.xpack.core.security.SecurityContext;
 import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesAction;
 import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesRequest;
 import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesResponse;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.permission.ResourcePrivileges;
 import org.elasticsearch.xpack.core.security.support.Exceptions;
 
 import java.util.Arrays;
@@ -328,7 +329,7 @@ public final class CcrLicenseChecker {
                 message.append(indices.length == 1 ? " index " : " indices ");
                 message.append(Arrays.toString(indices));
 
-                HasPrivilegesResponse.ResourcePrivileges resourcePrivileges = response.getIndexPrivileges().iterator().next();
+                ResourcePrivileges resourcePrivileges = response.getIndexPrivileges().iterator().next();
                 for (Map.Entry<String, Boolean> entry : resourcePrivileges.getPrivileges().entrySet()) {
                     if (entry.getValue() == false) {
                         message.append(", privilege for action [");

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java

@@ -137,6 +137,9 @@ import org.elasticsearch.xpack.core.rollup.job.RollupJobStatus;
 import org.elasticsearch.xpack.core.security.SecurityFeatureSetUsage;
 import org.elasticsearch.xpack.core.security.SecurityField;
 import org.elasticsearch.xpack.core.security.SecuritySettings;
+import org.elasticsearch.xpack.core.security.action.CreateApiKeyAction;
+import org.elasticsearch.xpack.core.security.action.GetApiKeyAction;
+import org.elasticsearch.xpack.core.security.action.InvalidateApiKeyAction;
 import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheAction;
 import org.elasticsearch.xpack.core.security.action.role.ClearRolesCacheAction;
 import org.elasticsearch.xpack.core.security.action.role.DeleteRoleAction;
@@ -314,6 +317,9 @@ public class XPackClientPlugin extends Plugin implements ActionPlugin, NetworkPl
                 InvalidateTokenAction.INSTANCE,
                 GetCertificateInfoAction.INSTANCE,
                 RefreshTokenAction.INSTANCE,
+                CreateApiKeyAction.INSTANCE,
+                InvalidateApiKeyAction.INSTANCE,
+                GetApiKeyAction.INSTANCE,
                 // upgrade
                 IndexUpgradeInfoAction.INSTANCE,
                 IndexUpgradeAction.INSTANCE,

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java

@@ -99,10 +99,14 @@ public class XPackSettings {
     public static final Setting<Boolean> RESERVED_REALM_ENABLED_SETTING = Setting.boolSetting("xpack.security.authc.reserved_realm.enabled",
             true, Setting.Property.NodeScope);
 
-    /** Setting for enabling or disabling the token service. Defaults to true */
+    /** Setting for enabling or disabling the token service. Defaults to the value of https being enabled */
     public static final Setting<Boolean> TOKEN_SERVICE_ENABLED_SETTING = Setting.boolSetting("xpack.security.authc.token.enabled",
         XPackSettings.HTTP_SSL_ENABLED::getRaw, Setting.Property.NodeScope);
 
+    /** Setting for enabling or disabling the api key service. Defaults to the value of https being enabled */
+    public static final Setting<Boolean> API_KEY_SERVICE_ENABLED_SETTING = Setting.boolSetting("xpack.security.authc.api_key.enabled",
+        XPackSettings.HTTP_SSL_ENABLED::getRaw, Setting.Property.NodeScope);
+
     /** Setting for enabling or disabling FIPS mode. Defaults to false */
     public static final Setting<Boolean> FIPS_MODE_ENABLED =
         Setting.boolSetting("xpack.security.fips_mode.enabled", false, Property.NodeScope);
@@ -199,6 +203,7 @@ public class XPackSettings {
         settings.add(HTTP_SSL_ENABLED);
         settings.add(RESERVED_REALM_ENABLED_SETTING);
         settings.add(TOKEN_SERVICE_ENABLED_SETTING);
+        settings.add(API_KEY_SERVICE_ENABLED_SETTING);
         settings.add(SQL_ENABLED);
         settings.add(USER_SETTING);
         settings.add(ROLLUP_ENABLED);

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityContext.java

@@ -13,9 +13,11 @@ import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.util.concurrent.ThreadContext.StoredContext;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authc.Authentication.AuthenticationType;
 import org.elasticsearch.xpack.core.security.user.User;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.Objects;
 import java.util.function.Consumer;
 
@@ -71,7 +73,8 @@ public class SecurityContext {
         } else {
             lookedUpBy = null;
         }
-        setAuthentication(new Authentication(user, authenticatedBy, lookedUpBy, version));
+        setAuthentication(
+            new Authentication(user, authenticatedBy, lookedUpBy, version, AuthenticationType.INTERNAL, Collections.emptyMap()));
     }
 
     /** Writes the authentication to the thread context */
@@ -89,7 +92,7 @@ public class SecurityContext {
      */
     public void executeAsUser(User user, Consumer<StoredContext> consumer, Version version) {
         final StoredContext original = threadContext.newStoredContext(true);
-        try (ThreadContext.StoredContext ctx = threadContext.stashContext()) {
+        try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
             setUser(user, version);
             consumer.accept(original);
         }
@@ -102,9 +105,9 @@ public class SecurityContext {
     public void executeAfterRewritingAuthentication(Consumer<StoredContext> consumer, Version version) {
         final StoredContext original = threadContext.newStoredContext(true);
         final Authentication authentication = Objects.requireNonNull(userSettings.getAuthentication());
-        try (ThreadContext.StoredContext ctx = threadContext.stashContext()) {
+        try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
             setAuthentication(new Authentication(authentication.getUser(), authentication.getAuthenticatedBy(),
-                                                 authentication.getLookedUpBy(), version));
+                authentication.getLookedUpBy(), version, authentication.getAuthenticationType(), authentication.getMetadata()));
             consumer.accept(original);
         }
     }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/ApiKey.java

@@ -0,0 +1,165 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.action;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
+/**
+ * API key information
+ */
+public final class ApiKey implements ToXContentObject, Writeable {
+
+    private final String name;
+    private final String id;
+    private final Instant creation;
+    private final Instant expiration;
+    private final boolean invalidated;
+    private final String username;
+    private final String realm;
+
+    public ApiKey(String name, String id, Instant creation, Instant expiration, boolean invalidated, String username, String realm) {
+        this.name = name;
+        this.id = id;
+        // As we do not yet support the nanosecond precision when we serialize to JSON,
+        // here creating the 'Instant' of milliseconds precision.
+        // This Instant can then be used for date comparison.
+        this.creation = Instant.ofEpochMilli(creation.toEpochMilli());
+        this.expiration = (expiration != null) ? Instant.ofEpochMilli(expiration.toEpochMilli()): null;
+        this.invalidated = invalidated;
+        this.username = username;
+        this.realm = realm;
+    }
+
+    public ApiKey(StreamInput in) throws IOException {
+        this.name = in.readString();
+        this.id = in.readString();
+        this.creation = in.readInstant();
+        this.expiration = in.readOptionalInstant();
+        this.invalidated = in.readBoolean();
+        this.username = in.readString();
+        this.realm = in.readString();
+    }
+
+    public String getId() {
+        return id;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public Instant getCreation() {
+        return creation;
+    }
+
+    public Instant getExpiration() {
+        return expiration;
+    }
+
+    public boolean isInvalidated() {
+        return invalidated;
+    }
+
+    public String getUsername() {
+        return username;
+    }
+
+    public String getRealm() {
+        return realm;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject()
+        .field("id", id)
+        .field("name", name)
+        .field("creation", creation.toEpochMilli());
+        if (expiration != null) {
+            builder.field("expiration", expiration.toEpochMilli());
+        }
+        builder.field("invalidated", invalidated)
+        .field("username", username)
+        .field("realm", realm);
+        return builder.endObject();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(name);
+        out.writeString(id);
+        out.writeInstant(creation);
+        out.writeOptionalInstant(expiration);
+        out.writeBoolean(invalidated);
+        out.writeString(username);
+        out.writeString(realm);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(name, id, creation, expiration, invalidated, username, realm);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null) {
+            return false;
+        }
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+        ApiKey other = (ApiKey) obj;
+        return Objects.equals(name, other.name)
+                && Objects.equals(id, other.id)
+                && Objects.equals(creation, other.creation)
+                && Objects.equals(expiration, other.expiration)
+                && Objects.equals(invalidated, other.invalidated)
+                && Objects.equals(username, other.username)
+                && Objects.equals(realm, other.realm);
+    }
+
+    static ConstructingObjectParser<ApiKey, Void> PARSER = new ConstructingObjectParser<>("api_key", args -> {
+        return new ApiKey((String) args[0], (String) args[1], Instant.ofEpochMilli((Long) args[2]),
+                (args[3] == null) ? null : Instant.ofEpochMilli((Long) args[3]), (Boolean) args[4], (String) args[5], (String) args[6]);
+    });
+    static {
+        PARSER.declareString(constructorArg(), new ParseField("name"));
+        PARSER.declareString(constructorArg(), new ParseField("id"));
+        PARSER.declareLong(constructorArg(), new ParseField("creation"));
+        PARSER.declareLong(optionalConstructorArg(), new ParseField("expiration"));
+        PARSER.declareBoolean(constructorArg(), new ParseField("invalidated"));
+        PARSER.declareString(constructorArg(), new ParseField("username"));
+        PARSER.declareString(constructorArg(), new ParseField("realm"));
+    }
+
+    public static ApiKey fromXContent(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
+
+    @Override
+    public String toString() {
+        return "ApiKey [name=" + name + ", id=" + id + ", creation=" + creation + ", expiration=" + expiration + ", invalidated="
+                + invalidated + ", username=" + username + ", realm=" + realm + "]";
+    }
+
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyAction.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.action;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.common.io.stream.Writeable;
+
+/**
+ * Action for the creation of an API key
+ */
+public final class CreateApiKeyAction extends Action<CreateApiKeyResponse> {
+
+    public static final String NAME = "cluster:admin/xpack/security/api_key/create";
+    public static final CreateApiKeyAction INSTANCE = new CreateApiKeyAction();
+
+    private CreateApiKeyAction() {
+        super(NAME);
+    }
+
+    @Override
+    public CreateApiKeyResponse newResponse() {
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+    }
+
+    @Override
+    public Writeable.Reader<CreateApiKeyResponse> getResponseReader() {
+        return CreateApiKeyResponse::new;
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequest.java

@@ -0,0 +1,132 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.action;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+/**
+ * Request class used for the creation of an API key. The request requires a name to be provided
+ * and optionally an expiration time and permission limitation can be provided.
+ */
+public final class CreateApiKeyRequest extends ActionRequest {
+    public static final WriteRequest.RefreshPolicy DEFAULT_REFRESH_POLICY = WriteRequest.RefreshPolicy.WAIT_UNTIL;
+
+    private String name;
+    private TimeValue expiration;
+    private List<RoleDescriptor> roleDescriptors = Collections.emptyList();
+    private WriteRequest.RefreshPolicy refreshPolicy = DEFAULT_REFRESH_POLICY;
+
+    public CreateApiKeyRequest() {}
+
+    /**
+     * Create API Key request constructor
+     * @param name name for the API key
+     * @param roleDescriptors list of {@link RoleDescriptor}s
+     * @param expiration to specify expiration for the API key
+     */
+    public CreateApiKeyRequest(String name, List<RoleDescriptor> roleDescriptors, @Nullable TimeValue expiration) {
+        if (Strings.hasText(name)) {
+            this.name = name;
+        } else {
+            throw new IllegalArgumentException("name must not be null or empty");
+        }
+        this.roleDescriptors = Objects.requireNonNull(roleDescriptors, "role descriptors may not be null");
+        this.expiration = expiration;
+    }
+
+    public CreateApiKeyRequest(StreamInput in) throws IOException {
+        super(in);
+        this.name = in.readString();
+        this.expiration = in.readOptionalTimeValue();
+        this.roleDescriptors = Collections.unmodifiableList(in.readList(RoleDescriptor::new));
+        this.refreshPolicy = WriteRequest.RefreshPolicy.readFrom(in);
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public void setName(String name) {
+        if (Strings.hasText(name)) {
+            this.name = name;
+        } else {
+            throw new IllegalArgumentException("name must not be null or empty");
+        }
+    }
+
+    public TimeValue getExpiration() {
+        return expiration;
+    }
+
+    public void setExpiration(TimeValue expiration) {
+        this.expiration = expiration;
+    }
+
+    public List<RoleDescriptor> getRoleDescriptors() {
+        return roleDescriptors;
+    }
+
+    public void setRoleDescriptors(List<RoleDescriptor> roleDescriptors) {
+        this.roleDescriptors = Collections.unmodifiableList(Objects.requireNonNull(roleDescriptors, "role descriptors may not be null"));
+    }
+
+    public WriteRequest.RefreshPolicy getRefreshPolicy() {
+        return refreshPolicy;
+    }
+
+    public void setRefreshPolicy(WriteRequest.RefreshPolicy refreshPolicy) {
+        this.refreshPolicy = Objects.requireNonNull(refreshPolicy, "refresh policy may not be null");
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        ActionRequestValidationException validationException = null;
+        if (Strings.isNullOrEmpty(name)) {
+            validationException = addValidationError("name is required", validationException);
+        } else {
+            if (name.length() > 256) {
+                validationException = addValidationError("name may not be more than 256 characters long", validationException);
+            }
+            if (name.equals(name.trim()) == false) {
+                validationException = addValidationError("name may not begin or end with whitespace", validationException);
+            }
+            if (name.startsWith("_")) {
+                validationException = addValidationError("name may not begin with an underscore", validationException);
+            }
+        }
+        return validationException;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeString(name);
+        out.writeOptionalTimeValue(expiration);
+        out.writeList(roleDescriptors);
+        refreshPolicy.writeTo(out);
+    }
+
+    @Override
+    public void readFrom(StreamInput in) {
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestBuilder.java

@@ -0,0 +1,84 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.security.action;
+
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
+/**
+ * Request builder for populating a {@link CreateApiKeyRequest}
+ */
+public final class CreateApiKeyRequestBuilder extends ActionRequestBuilder<CreateApiKeyRequest, CreateApiKeyResponse> {
+
+    @SuppressWarnings("unchecked")
+    static final ConstructingObjectParser<CreateApiKeyRequest, Void> PARSER = new ConstructingObjectParser<>(
+            "api_key_request", false, (args, v) -> {
+                return new CreateApiKeyRequest((String) args[0], (List<RoleDescriptor>) args[1],
+                        TimeValue.parseTimeValue((String) args[2], null, "expiration"));
+            });
+
+    static {
+        PARSER.declareString(constructorArg(), new ParseField("name"));
+        PARSER.declareNamedObjects(constructorArg(), (p, c, n) -> {
+            p.nextToken();
+            return RoleDescriptor.parse(n, p, false);
+        }, new ParseField("role_descriptors"));
+        PARSER.declareString(optionalConstructorArg(), new ParseField("expiration"));
+    }
+
+    public CreateApiKeyRequestBuilder(ElasticsearchClient client) {
+        super(client, CreateApiKeyAction.INSTANCE, new CreateApiKeyRequest());
+    }
+
+    public CreateApiKeyRequestBuilder setName(String name) {
+        request.setName(name);
+        return this;
+    }
+
+    public CreateApiKeyRequestBuilder setExpiration(TimeValue expiration) {
+        request.setExpiration(expiration);
+        return this;
+    }
+
+    public CreateApiKeyRequestBuilder setRoleDescriptors(List<RoleDescriptor> roleDescriptors) {
+        request.setRoleDescriptors(roleDescriptors);
+        return this;
+    }
+
+    public CreateApiKeyRequestBuilder setRefreshPolicy(WriteRequest.RefreshPolicy refreshPolicy) {
+        request.setRefreshPolicy(refreshPolicy);
+        return this;
+    }
+
+    public CreateApiKeyRequestBuilder source(BytesReference source, XContentType xContentType) throws IOException {
+        final NamedXContentRegistry registry = NamedXContentRegistry.EMPTY;
+        try (InputStream stream = source.streamInput();
+                XContentParser parser = xContentType.xContent().createParser(registry, LoggingDeprecationHandler.INSTANCE, stream)) {
+            CreateApiKeyRequest createApiKeyRequest = PARSER.parse(parser, null);
+            setName(createApiKeyRequest.getName());
+            setRoleDescriptors(createApiKeyRequest.getRoleDescriptors());
+            setExpiration(createApiKeyRequest.getExpiration());
+        }
+        return this;
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyResponse.java

@@ -0,0 +1,168 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.action;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.CharArrays;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.Arrays;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
+/**
+ * Response for the successful creation of an api key
+ */
+public final class CreateApiKeyResponse extends ActionResponse implements ToXContentObject {
+
+    static ConstructingObjectParser<CreateApiKeyResponse, Void> PARSER = new ConstructingObjectParser<>("create_api_key_response",
+            args -> new CreateApiKeyResponse((String) args[0], (String) args[1], new SecureString((String) args[2]),
+                    (args[3] == null) ? null : Instant.ofEpochMilli((Long) args[3])));
+    static {
+        PARSER.declareString(constructorArg(), new ParseField("name"));
+        PARSER.declareString(constructorArg(), new ParseField("id"));
+        PARSER.declareString(constructorArg(), new ParseField("api_key"));
+        PARSER.declareLong(optionalConstructorArg(), new ParseField("expiration"));
+    }
+
+    private final String name;
+    private final String id;
+    private final SecureString key;
+    private final Instant expiration;
+
+    public CreateApiKeyResponse(String name, String id, SecureString key, Instant expiration) {
+        this.name = name;
+        this.id = id;
+        this.key = key;
+        // As we do not yet support the nanosecond precision when we serialize to JSON,
+        // here creating the 'Instant' of milliseconds precision.
+        // This Instant can then be used for date comparison.
+        this.expiration = (expiration != null) ? Instant.ofEpochMilli(expiration.toEpochMilli()): null;
+    }
+
+    public CreateApiKeyResponse(StreamInput in) throws IOException {
+        super(in);
+        this.name = in.readString();
+        this.id = in.readString();
+        byte[] bytes = null;
+        try {
+            bytes = in.readByteArray();
+            this.key = new SecureString(CharArrays.utf8BytesToChars(bytes));
+        } finally {
+            if (bytes != null) {
+                Arrays.fill(bytes, (byte) 0);
+            }
+        }
+        this.expiration = in.readOptionalInstant();
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public String getId() {
+        return id;
+    }
+
+    public SecureString getKey() {
+        return key;
+    }
+
+    @Nullable
+    public Instant getExpiration() {
+        return expiration;
+    }
+
+    @Override
+    public int hashCode() {
+        final int prime = 31;
+        int result = 1;
+        result = prime * result + ((expiration == null) ? 0 : expiration.hashCode());
+        result = prime * result + Objects.hash(id, name, key);
+        return result;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+        final CreateApiKeyResponse other = (CreateApiKeyResponse) obj;
+        if (expiration == null) {
+            if (other.expiration != null)
+                return false;
+        } else if (!Objects.equals(expiration, other.expiration))
+            return false;
+        return Objects.equals(id, other.id)
+                && Objects.equals(key, other.key)
+                && Objects.equals(name, other.name);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeString(name);
+        out.writeString(id);
+        byte[] bytes = null;
+        try {
+            bytes = CharArrays.toUtf8Bytes(key.getChars());
+            out.writeByteArray(bytes);
+        } finally {
+            if (bytes != null) {
+                Arrays.fill(bytes, (byte) 0);
+            }
+        }
+        out.writeOptionalInstant(expiration);
+    }
+
+    @Override
+    public void readFrom(StreamInput in) {
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+    }
+
+    public static CreateApiKeyResponse fromXContent(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject()
+            .field("id", id)
+            .field("name", name);
+        if (expiration != null) {
+            builder.field("expiration", expiration.toEpochMilli());
+        }
+        byte[] charBytes = CharArrays.toUtf8Bytes(key.getChars());
+        try {
+            builder.field("api_key").utf8Value(charBytes, 0, charBytes.length);
+        } finally {
+            Arrays.fill(charBytes, (byte) 0);
+        }
+        return builder.endObject();
+    }
+
+    @Override
+    public String toString() {
+        return "CreateApiKeyResponse [name=" + name + ", id=" + id + ", expiration=" + expiration + "]";
+    }
+
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GetApiKeyAction.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.action;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.common.io.stream.Writeable;
+
+/**
+ * Action for retrieving API key(s)
+ */
+public final class GetApiKeyAction extends Action<GetApiKeyResponse> {
+
+    public static final String NAME = "cluster:admin/xpack/security/api_key/get";
+    public static final GetApiKeyAction INSTANCE = new GetApiKeyAction();
+
+    private GetApiKeyAction() {
+        super(NAME);
+    }
+
+    @Override
+    public GetApiKeyResponse newResponse() {
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+    }
+
+    @Override
+    public Writeable.Reader<GetApiKeyResponse> getResponseReader() {
+        return GetApiKeyResponse::new;
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GetApiKeyRequest.java

@@ -0,0 +1,146 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.action;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+/**
+ * Request for get API key
+ */
+public final class GetApiKeyRequest extends ActionRequest {
+
+    private final String realmName;
+    private final String userName;
+    private final String apiKeyId;
+    private final String apiKeyName;
+
+    public GetApiKeyRequest() {
+        this(null, null, null, null);
+    }
+
+    public GetApiKeyRequest(StreamInput in) throws IOException {
+        super(in);
+        realmName = in.readOptionalString();
+        userName = in.readOptionalString();
+        apiKeyId = in.readOptionalString();
+        apiKeyName = in.readOptionalString();
+    }
+
+    public GetApiKeyRequest(@Nullable String realmName, @Nullable String userName, @Nullable String apiKeyId,
+            @Nullable String apiKeyName) {
+        this.realmName = realmName;
+        this.userName = userName;
+        this.apiKeyId = apiKeyId;
+        this.apiKeyName = apiKeyName;
+    }
+
+    public String getRealmName() {
+        return realmName;
+    }
+
+    public String getUserName() {
+        return userName;
+    }
+
+    public String getApiKeyId() {
+        return apiKeyId;
+    }
+
+    public String getApiKeyName() {
+        return apiKeyName;
+    }
+
+    /**
+     * Creates get API key request for given realm name
+     * @param realmName realm name
+     * @return {@link GetApiKeyRequest}
+     */
+    public static GetApiKeyRequest usingRealmName(String realmName) {
+        return new GetApiKeyRequest(realmName, null, null, null);
+    }
+
+    /**
+     * Creates get API key request for given user name
+     * @param userName user name
+     * @return {@link GetApiKeyRequest}
+     */
+    public static GetApiKeyRequest usingUserName(String userName) {
+        return new GetApiKeyRequest(null, userName, null, null);
+    }
+
+    /**
+     * Creates get API key request for given realm and user name
+     * @param realmName realm name
+     * @param userName user name
+     * @return {@link GetApiKeyRequest}
+     */
+    public static GetApiKeyRequest usingRealmAndUserName(String realmName, String userName) {
+        return new GetApiKeyRequest(realmName, userName, null, null);
+    }
+
+    /**
+     * Creates get API key request for given api key id
+     * @param apiKeyId api key id
+     * @return {@link GetApiKeyRequest}
+     */
+    public static GetApiKeyRequest usingApiKeyId(String apiKeyId) {
+        return new GetApiKeyRequest(null, null, apiKeyId, null);
+    }
+
+    /**
+     * Creates get api key request for given api key name
+     * @param apiKeyName api key name
+     * @return {@link GetApiKeyRequest}
+     */
+    public static GetApiKeyRequest usingApiKeyName(String apiKeyName) {
+        return new GetApiKeyRequest(null, null, null, apiKeyName);
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        ActionRequestValidationException validationException = null;
+        if (Strings.hasText(realmName) == false && Strings.hasText(userName) == false && Strings.hasText(apiKeyId) == false
+                && Strings.hasText(apiKeyName) == false) {
+            validationException = addValidationError("One of [api key id, api key name, username, realm name] must be specified",
+                    validationException);
+        }
+        if (Strings.hasText(apiKeyId) || Strings.hasText(apiKeyName)) {
+            if (Strings.hasText(realmName) || Strings.hasText(userName)) {
+                validationException = addValidationError(
+                        "username or realm name must not be specified when the api key id or api key name is specified",
+                        validationException);
+            }
+        }
+        if (Strings.hasText(apiKeyId) && Strings.hasText(apiKeyName)) {
+            validationException = addValidationError("only one of [api key id, api key name] can be specified", validationException);
+        }
+        return validationException;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeOptionalString(realmName);
+        out.writeOptionalString(userName);
+        out.writeOptionalString(apiKeyId);
+        out.writeOptionalString(apiKeyName);
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GetApiKeyResponse.java

@@ -0,0 +1,88 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.action;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
+/**
+ * Response for get API keys.<br>
+ * The result contains information about the API keys that were found.
+ */
+public final class GetApiKeyResponse extends ActionResponse implements ToXContentObject, Writeable {
+
+    private final ApiKey[] foundApiKeysInfo;
+
+    public GetApiKeyResponse(StreamInput in) throws IOException {
+        super(in);
+        this.foundApiKeysInfo = in.readArray(ApiKey::new, ApiKey[]::new);
+    }
+
+    public GetApiKeyResponse(Collection<ApiKey> foundApiKeysInfo) {
+        Objects.requireNonNull(foundApiKeysInfo, "found_api_keys_info must be provided");
+        this.foundApiKeysInfo = foundApiKeysInfo.toArray(new ApiKey[0]);
+    }
+
+    public static GetApiKeyResponse emptyResponse() {
+        return new GetApiKeyResponse(Collections.emptyList());
+    }
+
+    public ApiKey[] getApiKeyInfos() {
+        return foundApiKeysInfo;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject()
+            .array("api_keys", (Object[]) foundApiKeysInfo);
+        return builder.endObject();
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeArray(foundApiKeysInfo);
+    }
+
+    @SuppressWarnings("unchecked")
+    static ConstructingObjectParser<GetApiKeyResponse, Void> PARSER = new ConstructingObjectParser<>("get_api_key_response", args -> {
+        return (args[0] == null) ? GetApiKeyResponse.emptyResponse() : new GetApiKeyResponse((List<ApiKey>) args[0]);
+    });
+    static {
+        PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> ApiKey.fromXContent(p), new ParseField("api_keys"));
+    }
+
+    public static GetApiKeyResponse fromXContent(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
+
+    @Override
+    public String toString() {
+        return "GetApiKeyResponse [foundApiKeysInfo=" + foundApiKeysInfo + "]";
+    }
+
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyAction.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.action;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.common.io.stream.Writeable;
+
+/**
+ * Action for invalidating API key
+ */
+public final class InvalidateApiKeyAction extends Action<InvalidateApiKeyResponse> {
+
+    public static final String NAME = "cluster:admin/xpack/security/api_key/invalidate";
+    public static final InvalidateApiKeyAction INSTANCE = new InvalidateApiKeyAction();
+
+    private InvalidateApiKeyAction() {
+        super(NAME);
+    }
+
+    @Override
+    public InvalidateApiKeyResponse newResponse() {
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+    }
+
+    @Override
+    public Writeable.Reader<InvalidateApiKeyResponse> getResponseReader() {
+        return InvalidateApiKeyResponse::new;
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyRequest.java

@@ -0,0 +1,146 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.action;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+/**
+ * Request for invalidating API key(s) so that it can no longer be used
+ */
+public final class InvalidateApiKeyRequest extends ActionRequest {
+
+    private final String realmName;
+    private final String userName;
+    private final String id;
+    private final String name;
+
+    public InvalidateApiKeyRequest() {
+        this(null, null, null, null);
+    }
+
+    public InvalidateApiKeyRequest(StreamInput in) throws IOException {
+        super(in);
+        realmName = in.readOptionalString();
+        userName = in.readOptionalString();
+        id = in.readOptionalString();
+        name = in.readOptionalString();
+    }
+
+    public InvalidateApiKeyRequest(@Nullable String realmName, @Nullable String userName, @Nullable String id,
+            @Nullable String name) {
+        this.realmName = realmName;
+        this.userName = userName;
+        this.id = id;
+        this.name = name;
+    }
+
+    public String getRealmName() {
+        return realmName;
+    }
+
+    public String getUserName() {
+        return userName;
+    }
+
+    public String getId() {
+        return id;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    /**
+     * Creates invalidate api key request for given realm name
+     * @param realmName realm name
+     * @return {@link InvalidateApiKeyRequest}
+     */
+    public static InvalidateApiKeyRequest usingRealmName(String realmName) {
+        return new InvalidateApiKeyRequest(realmName, null, null, null);
+    }
+
+    /**
+     * Creates invalidate API key request for given user name
+     * @param userName user name
+     * @return {@link InvalidateApiKeyRequest}
+     */
+    public static InvalidateApiKeyRequest usingUserName(String userName) {
+        return new InvalidateApiKeyRequest(null, userName, null, null);
+    }
+
+    /**
+     * Creates invalidate API key request for given realm and user name
+     * @param realmName realm name
+     * @param userName user name
+     * @return {@link InvalidateApiKeyRequest}
+     */
+    public static InvalidateApiKeyRequest usingRealmAndUserName(String realmName, String userName) {
+        return new InvalidateApiKeyRequest(realmName, userName, null, null);
+    }
+
+    /**
+     * Creates invalidate API key request for given api key id
+     * @param id api key id
+     * @return {@link InvalidateApiKeyRequest}
+     */
+    public static InvalidateApiKeyRequest usingApiKeyId(String id) {
+        return new InvalidateApiKeyRequest(null, null, id, null);
+    }
+
+    /**
+     * Creates invalidate api key request for given api key name
+     * @param name api key name
+     * @return {@link InvalidateApiKeyRequest}
+     */
+    public static InvalidateApiKeyRequest usingApiKeyName(String name) {
+        return new InvalidateApiKeyRequest(null, null, null, name);
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        ActionRequestValidationException validationException = null;
+        if (Strings.hasText(realmName) == false && Strings.hasText(userName) == false && Strings.hasText(id) == false
+                && Strings.hasText(name) == false) {
+            validationException = addValidationError("One of [api key id, api key name, username, realm name] must be specified",
+                    validationException);
+        }
+        if (Strings.hasText(id) || Strings.hasText(name)) {
+            if (Strings.hasText(realmName) || Strings.hasText(userName)) {
+                validationException = addValidationError(
+                        "username or realm name must not be specified when the api key id or api key name is specified",
+                        validationException);
+            }
+        }
+        if (Strings.hasText(id) && Strings.hasText(name)) {
+            validationException = addValidationError("only one of [api key id, api key name] can be specified", validationException);
+        }
+        return validationException;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeOptionalString(realmName);
+        out.writeOptionalString(userName);
+        out.writeOptionalString(id);
+        out.writeOptionalString(name);
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyResponse.java

@@ -0,0 +1,141 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.action;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
+/**
+ * Response for invalidation of one or more API keys result.<br>
+ * The result contains information about:
+ * <ul>
+ * <li>API key ids that were actually invalidated</li>
+ * <li>API key ids that were not invalidated in this request because they were already invalidated</li>
+ * <li>how many errors were encountered while invalidating API keys and the error details</li>
+ * </ul>
+ */
+public final class InvalidateApiKeyResponse extends ActionResponse implements ToXContentObject, Writeable {
+
+    private final List<String> invalidatedApiKeys;
+    private final List<String> previouslyInvalidatedApiKeys;
+    private final List<ElasticsearchException> errors;
+
+    public InvalidateApiKeyResponse(StreamInput in) throws IOException {
+        super(in);
+        this.invalidatedApiKeys = in.readList(StreamInput::readString);
+        this.previouslyInvalidatedApiKeys = in.readList(StreamInput::readString);
+        this.errors = in.readList(StreamInput::readException);
+    }
+
+    /**
+     * Constructor for API keys invalidation response
+     * @param invalidatedApiKeys list of invalidated API key ids
+     * @param previouslyInvalidatedApiKeys list of previously invalidated API key ids
+     * @param errors list of encountered errors while invalidating API keys
+     */
+    public InvalidateApiKeyResponse(List<String> invalidatedApiKeys, List<String> previouslyInvalidatedApiKeys,
+                                    @Nullable List<ElasticsearchException> errors) {
+        this.invalidatedApiKeys = Objects.requireNonNull(invalidatedApiKeys, "invalidated_api_keys must be provided");
+        this.previouslyInvalidatedApiKeys = Objects.requireNonNull(previouslyInvalidatedApiKeys,
+                "previously_invalidated_api_keys must be provided");
+        if (null != errors) {
+            this.errors = errors;
+        } else {
+            this.errors = Collections.emptyList();
+        }
+    }
+
+    public static InvalidateApiKeyResponse emptyResponse() {
+        return new InvalidateApiKeyResponse(Collections.emptyList(), Collections.emptyList(), Collections.emptyList());
+    }
+
+    public List<String> getInvalidatedApiKeys() {
+        return invalidatedApiKeys;
+    }
+
+    public List<String> getPreviouslyInvalidatedApiKeys() {
+        return previouslyInvalidatedApiKeys;
+    }
+
+    public List<ElasticsearchException> getErrors() {
+        return errors;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject()
+            .array("invalidated_api_keys", invalidatedApiKeys.toArray(Strings.EMPTY_ARRAY))
+            .array("previously_invalidated_api_keys", previouslyInvalidatedApiKeys.toArray(Strings.EMPTY_ARRAY))
+            .field("error_count", errors.size());
+        if (errors.isEmpty() == false) {
+            builder.field("error_details");
+            builder.startArray();
+            for (ElasticsearchException e : errors) {
+                builder.startObject();
+                ElasticsearchException.generateThrowableXContent(builder, params, e);
+                builder.endObject();
+            }
+            builder.endArray();
+        }
+        return builder.endObject();
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeStringCollection(invalidatedApiKeys);
+        out.writeStringCollection(previouslyInvalidatedApiKeys);
+        out.writeCollection(errors, StreamOutput::writeException);
+    }
+
+    static ConstructingObjectParser<InvalidateApiKeyResponse, Void> PARSER = new ConstructingObjectParser<>("invalidate_api_key_response",
+            args -> {
+                return new InvalidateApiKeyResponse((List<String>) args[0], (List<String>) args[1], (List<ElasticsearchException>) args[3]);
+            });
+    static {
+        PARSER.declareStringArray(constructorArg(), new ParseField("invalidated_api_keys"));
+        PARSER.declareStringArray(constructorArg(), new ParseField("previously_invalidated_api_keys"));
+        // we parse error_count but ignore it while constructing response
+        PARSER.declareInt(constructorArg(), new ParseField("error_count"));
+        PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> ElasticsearchException.fromXContent(p),
+                new ParseField("error_details"));
+    }
+
+    public static InvalidateApiKeyResponse fromXContent(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
+
+    @Override
+    public String toString() {
+        return "InvalidateApiKeyResponse [invalidatedApiKeys=" + invalidatedApiKeys + ", previouslyInvalidatedApiKeys="
+                + previouslyInvalidatedApiKeys + ", errors=" + errors + "]";
+    }
+
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/GetRolesResponse.java

@@ -37,7 +37,7 @@ public class GetRolesResponse extends ActionResponse {
         int size = in.readVInt();
         roles = new RoleDescriptor[size];
         for (int i = 0; i < size; i++) {
-            roles[i] = RoleDescriptor.readFrom(in);
+            roles[i] = new RoleDescriptor(in);
         }
     }
 
@@ -46,7 +46,7 @@ public class GetRolesResponse extends ActionResponse {
         super.writeTo(out);
         out.writeVInt(roles.length);
         for (RoleDescriptor role : roles) {
-            RoleDescriptor.writeTo(role, out);
+            role.writeTo(out);
         }
     }
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesResponse.java

@@ -11,6 +11,7 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.core.security.authz.permission.ResourcePrivileges;
 
 import java.io.IOException;
 import java.util.Collection;
@@ -49,7 +50,7 @@ public class HasPrivilegesResponse extends ActionResponse implements ToXContentO
     }
 
     private static Set<ResourcePrivileges> sorted(Collection<ResourcePrivileges> resources) {
-        final Set<ResourcePrivileges> set = new TreeSet<>(Comparator.comparing(o -> o.resource));
+        final Set<ResourcePrivileges> set = new TreeSet<>(Comparator.comparing(o -> o.getResource()));
         set.addAll(resources);
         return set;
     }
@@ -116,11 +117,11 @@ public class HasPrivilegesResponse extends ActionResponse implements ToXContentO
 
     private static Set<ResourcePrivileges> readResourcePrivileges(StreamInput in) throws IOException {
         final int count = in.readVInt();
-        final Set<ResourcePrivileges> set = new TreeSet<>(Comparator.comparing(o -> o.resource));
+        final Set<ResourcePrivileges> set = new TreeSet<>(Comparator.comparing(o -> o.getResource()));
         for (int i = 0; i < count; i++) {
             final String index = in.readString();
             final Map<String, Boolean> privileges = in.readMap(StreamInput::readString, StreamInput::readBoolean);
-            set.add(new ResourcePrivileges(index, privileges));
+            set.add(ResourcePrivileges.builder(index).addPrivileges(privileges).build());
         }
         return set;
     }
@@ -144,8 +145,8 @@ public class HasPrivilegesResponse extends ActionResponse implements ToXContentO
     private static void writeResourcePrivileges(StreamOutput out, Set<ResourcePrivileges> privileges) throws IOException {
         out.writeVInt(privileges.size());
         for (ResourcePrivileges priv : privileges) {
-            out.writeString(priv.resource);
-            out.writeMap(priv.privileges, StreamOutput::writeString, StreamOutput::writeBoolean);
+            out.writeString(priv.getResource());
+            out.writeMap(priv.getPrivileges(), StreamOutput::writeString, StreamOutput::writeBoolean);
         }
     }
 
@@ -181,60 +182,14 @@ public class HasPrivilegesResponse extends ActionResponse implements ToXContentO
         return builder;
     }
 
-    private void appendResources(XContentBuilder builder, String field, Set<HasPrivilegesResponse.ResourcePrivileges> privileges)
+    private void appendResources(XContentBuilder builder, String field, Set<ResourcePrivileges> privileges)
         throws IOException {
         builder.startObject(field);
-        for (HasPrivilegesResponse.ResourcePrivileges privilege : privileges) {
+        for (ResourcePrivileges privilege : privileges) {
             builder.field(privilege.getResource());
             builder.map(privilege.getPrivileges());
         }
         builder.endObject();
     }
 
-
-    public static class ResourcePrivileges {
-        private final String resource;
-        private final Map<String, Boolean> privileges;
-
-        public ResourcePrivileges(String resource, Map<String, Boolean> privileges) {
-            this.resource = Objects.requireNonNull(resource);
-            this.privileges = Collections.unmodifiableMap(privileges);
-        }
-
-        public String getResource() {
-            return resource;
-        }
-
-        public Map<String, Boolean> getPrivileges() {
-            return privileges;
-        }
-
-        @Override
-        public String toString() {
-            return getClass().getSimpleName() + "{" +
-                    "resource='" + resource + '\'' +
-                    ", privileges=" + privileges +
-                    '}';
-        }
-
-        @Override
-        public int hashCode() {
-            int result = resource.hashCode();
-            result = 31 * result + privileges.hashCode();
-            return result;
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) {
-                return true;
-            }
-            if (o == null || getClass() != o.getClass()) {
-                return false;
-            }
-
-            final ResourcePrivileges other = (ResourcePrivileges) o;
-            return this.resource.equals(other.resource) && this.privileges.equals(other.privileges);
-        }
-    }
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java

@@ -18,6 +18,8 @@ import org.elasticsearch.xpack.core.security.user.User;
 
 import java.io.IOException;
 import java.util.Base64;
+import java.util.Collections;
+import java.util.Map;
 import java.util.Objects;
 
 // TODO(hub-cap) Clean this up after moving User over - This class can re-inherit its field AUTHENTICATION_KEY in AuthenticationField.
@@ -28,16 +30,25 @@ public class Authentication implements ToXContentObject {
     private final RealmRef authenticatedBy;
     private final RealmRef lookedUpBy;
     private final Version version;
+    private final AuthenticationType type;
+    private final Map<String, Object> metadata;
 
     public Authentication(User user, RealmRef authenticatedBy, RealmRef lookedUpBy) {
         this(user, authenticatedBy, lookedUpBy, Version.CURRENT);
     }
 
     public Authentication(User user, RealmRef authenticatedBy, RealmRef lookedUpBy, Version version) {
+        this(user, authenticatedBy, lookedUpBy, version, AuthenticationType.REALM, Collections.emptyMap());
+    }
+
+    public Authentication(User user, RealmRef authenticatedBy, RealmRef lookedUpBy, Version version,
+                          AuthenticationType type, Map<String, Object> metadata) {
         this.user = Objects.requireNonNull(user);
         this.authenticatedBy = Objects.requireNonNull(authenticatedBy);
         this.lookedUpBy = lookedUpBy;
         this.version = version;
+        this.type = type;
+        this.metadata = metadata;
     }
 
     public Authentication(StreamInput in) throws IOException {
@@ -49,6 +60,13 @@ public class Authentication implements ToXContentObject {
             this.lookedUpBy = null;
         }
         this.version = in.getVersion();
+        if (in.getVersion().onOrAfter(Version.V_7_0_0)) { // TODO change to V6_6 after backport
+            type = AuthenticationType.values()[in.readVInt()];
+            metadata = in.readMap();
+        } else {
+            type = AuthenticationType.REALM;
+            metadata = Collections.emptyMap();
+        }
     }
 
     public User getUser() {
@@ -67,8 +85,15 @@ public class Authentication implements ToXContentObject {
         return version;
     }
 
-    public static Authentication readFromContext(ThreadContext ctx)
-            throws IOException, IllegalArgumentException {
+    public AuthenticationType getAuthenticationType() {
+        return type;
+    }
+
+    public Map<String, Object> getMetadata() {
+        return metadata;
+    }
+
+    public static Authentication readFromContext(ThreadContext ctx) throws IOException, IllegalArgumentException {
         Authentication authentication = ctx.getTransient(AuthenticationField.AUTHENTICATION_KEY);
         if (authentication != null) {
             assert ctx.getHeader(AuthenticationField.AUTHENTICATION_KEY) != null;
@@ -107,8 +132,7 @@ public class Authentication implements ToXContentObject {
      * Writes the authentication to the context. There must not be an existing authentication in the context and if there is an
      * {@link IllegalStateException} will be thrown
      */
-    public void writeToContext(ThreadContext ctx)
-            throws IOException, IllegalArgumentException {
+    public void writeToContext(ThreadContext ctx) throws IOException, IllegalArgumentException {
         ensureContextDoesNotContainAuthentication(ctx);
         String header = encode();
         ctx.putTransient(AuthenticationField.AUTHENTICATION_KEY, this);
@@ -141,28 +165,28 @@ public class Authentication implements ToXContentObject {
         } else {
             out.writeBoolean(false);
         }
+        if (out.getVersion().onOrAfter(Version.V_7_0_0)) { // TODO change to V6_6 after backport
+            out.writeVInt(type.ordinal());
+            out.writeMap(metadata);
+        }
     }
 
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
-
         Authentication that = (Authentication) o;
-
-        if (!user.equals(that.user)) return false;
-        if (!authenticatedBy.equals(that.authenticatedBy)) return false;
-        if (lookedUpBy != null ? !lookedUpBy.equals(that.lookedUpBy) : that.lookedUpBy != null) return false;
-        return version.equals(that.version);
+        return user.equals(that.user) &&
+            authenticatedBy.equals(that.authenticatedBy) &&
+            Objects.equals(lookedUpBy, that.lookedUpBy) &&
+            version.equals(that.version) &&
+            type == that.type &&
+            metadata.equals(that.metadata);
     }
 
     @Override
     public int hashCode() {
-        int result = user.hashCode();
-        result = 31 * result + authenticatedBy.hashCode();
-        result = 31 * result + (lookedUpBy != null ? lookedUpBy.hashCode() : 0);
-        result = 31 * result + version.hashCode();
-        return result;
+        return Objects.hash(user, authenticatedBy, lookedUpBy, version, type, metadata);
     }
 
     @Override
@@ -246,5 +270,13 @@ public class Authentication implements ToXContentObject {
             return result;
         }
     }
+
+    public enum AuthenticationType {
+        REALM,
+        API_KEY,
+        TOKEN,
+        ANONYMOUS,
+        INTERNAL
+    }
 }
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandler.java

@@ -68,10 +68,12 @@ public class DefaultAuthenticationFailureHandler implements AuthenticationFailur
             return 0;
         } else if (headerValue.regionMatches(true, 0, "bearer", 0, "bearer".length())) {
             return 1;
-        } else if (headerValue.regionMatches(true, 0, "basic", 0, "basic".length())) {
+        } else if (headerValue.regionMatches(true, 0, "apikey", 0, "apikey".length())) {
             return 2;
-        } else {
+        } else if (headerValue.regionMatches(true, 0, "basic", 0, "basic".length())) {
             return 3;
+        } else {
+            return 4;
         }
     }
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java

@@ -43,7 +43,7 @@ import java.util.Objects;
  * A holder for a Role that contains user-readable information about the Role
  * without containing the actual Role object.
  */
-public class RoleDescriptor implements ToXContentObject {
+public class RoleDescriptor implements ToXContentObject, Writeable {
 
     public static final String ROLE_TYPE = "role";
 
@@ -110,6 +110,27 @@ public class RoleDescriptor implements ToXContentObject {
                 Collections.singletonMap("enabled", true);
     }
 
+    public RoleDescriptor(StreamInput in) throws IOException {
+        this.name = in.readString();
+        this.clusterPrivileges = in.readStringArray();
+        int size = in.readVInt();
+        this.indicesPrivileges = new IndicesPrivileges[size];
+        for (int i = 0; i < size; i++) {
+            indicesPrivileges[i] = new IndicesPrivileges(in);
+        }
+        this.runAs = in.readStringArray();
+        this.metadata = in.readMap();
+        this.transientMetadata = in.readMap();
+
+        if (in.getVersion().onOrAfter(Version.V_6_4_0)) {
+            this.applicationPrivileges = in.readArray(ApplicationResourcePrivileges::new, ApplicationResourcePrivileges[]::new);
+            this.conditionalClusterPrivileges = ConditionalClusterPrivileges.readArray(in);
+        } else {
+            this.applicationPrivileges = ApplicationResourcePrivileges.NONE;
+            this.conditionalClusterPrivileges = ConditionalClusterPrivileges.EMPTY_ARRAY;
+        }
+    }
+
     public String getName() {
         return this.name;
     }
@@ -232,46 +253,20 @@ public class RoleDescriptor implements ToXContentObject {
         return builder.endObject();
     }
 
-    public static RoleDescriptor readFrom(StreamInput in) throws IOException {
-        String name = in.readString();
-        String[] clusterPrivileges = in.readStringArray();
-        int size = in.readVInt();
-        IndicesPrivileges[] indicesPrivileges = new IndicesPrivileges[size];
-        for (int i = 0; i < size; i++) {
-            indicesPrivileges[i] = new IndicesPrivileges(in);
-        }
-        String[] runAs = in.readStringArray();
-        Map<String, Object> metadata = in.readMap();
-
-        final Map<String, Object> transientMetadata = in.readMap();
-
-        final ApplicationResourcePrivileges[] applicationPrivileges;
-        final ConditionalClusterPrivilege[] conditionalClusterPrivileges;
-        if (in.getVersion().onOrAfter(Version.V_6_4_0)) {
-            applicationPrivileges = in.readArray(ApplicationResourcePrivileges::new, ApplicationResourcePrivileges[]::new);
-            conditionalClusterPrivileges = ConditionalClusterPrivileges.readArray(in);
-        } else {
-            applicationPrivileges = ApplicationResourcePrivileges.NONE;
-            conditionalClusterPrivileges = ConditionalClusterPrivileges.EMPTY_ARRAY;
-        }
-
-        return new RoleDescriptor(name, clusterPrivileges, indicesPrivileges, applicationPrivileges, conditionalClusterPrivileges,
-            runAs, metadata, transientMetadata);
-    }
-
-    public static void writeTo(RoleDescriptor descriptor, StreamOutput out) throws IOException {
-        out.writeString(descriptor.name);
-        out.writeStringArray(descriptor.clusterPrivileges);
-        out.writeVInt(descriptor.indicesPrivileges.length);
-        for (IndicesPrivileges group : descriptor.indicesPrivileges) {
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(name);
+        out.writeStringArray(clusterPrivileges);
+        out.writeVInt(indicesPrivileges.length);
+        for (IndicesPrivileges group : indicesPrivileges) {
             group.writeTo(out);
         }
-        out.writeStringArray(descriptor.runAs);
-        out.writeMap(descriptor.metadata);
-        out.writeMap(descriptor.transientMetadata);
+        out.writeStringArray(runAs);
+        out.writeMap(metadata);
+        out.writeMap(transientMetadata);
         if (out.getVersion().onOrAfter(Version.V_6_4_0)) {
-            out.writeArray(ApplicationResourcePrivileges::write, descriptor.applicationPrivileges);
-            ConditionalClusterPrivileges.writeArray(out, descriptor.getConditionalClusterPrivileges());
+            out.writeArray(ApplicationResourcePrivileges::write, applicationPrivileges);
+            ConditionalClusterPrivileges.writeArray(out, getConditionalClusterPrivileges());
         }
     }
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/IndicesAccessControl.java

@@ -6,11 +6,13 @@
 package org.elasticsearch.xpack.core.security.authz.accesscontrol;
 
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.xpack.core.security.authz.IndicesAndAliasesResolverField;
+import org.elasticsearch.xpack.core.security.authz.permission.DocumentPermissions;
 import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissions;
 
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
 
@@ -22,7 +24,7 @@ public class IndicesAccessControl {
     public static final IndicesAccessControl ALLOW_ALL = new IndicesAccessControl(true, Collections.emptyMap());
     public static final IndicesAccessControl ALLOW_NO_INDICES = new IndicesAccessControl(true,
             Collections.singletonMap(IndicesAndAliasesResolverField.NO_INDEX_PLACEHOLDER,
-                    new IndicesAccessControl.IndexAccessControl(true, new FieldPermissions(), null)));
+                    new IndicesAccessControl.IndexAccessControl(true, new FieldPermissions(), DocumentPermissions.allowAll())));
 
     private final boolean granted;
     private final Map<String, IndexAccessControl> indexPermissions;
@@ -55,12 +57,12 @@ public class IndicesAccessControl {
 
         private final boolean granted;
         private final FieldPermissions fieldPermissions;
-        private final Set<BytesReference> queries;
+        private final DocumentPermissions documentPermissions;
 
-        public IndexAccessControl(boolean granted, FieldPermissions fieldPermissions, Set<BytesReference> queries) {
+        public IndexAccessControl(boolean granted, FieldPermissions fieldPermissions, DocumentPermissions documentPermissions) {
             this.granted = granted;
-            this.fieldPermissions = fieldPermissions;
-            this.queries = queries;
+            this.fieldPermissions = (fieldPermissions == null) ? FieldPermissions.DEFAULT : fieldPermissions;
+            this.documentPermissions = (documentPermissions == null) ? DocumentPermissions.allowAll() : documentPermissions;
         }
 
         /**
@@ -82,8 +84,33 @@ public class IndicesAccessControl {
          *         then this means that there are no document level restrictions
          */
         @Nullable
-        public Set<BytesReference> getQueries() {
-            return queries;
+        public DocumentPermissions getDocumentPermissions() {
+            return documentPermissions;
+        }
+
+        /**
+         * Returns a instance of {@link IndexAccessControl}, where the privileges for {@code this} object are constrained by the privileges
+         * contained in the provided parameter.<br>
+         * Allowed fields for this index permission would be an intersection of allowed fields.<br>
+         * Allowed documents for this index permission would be an intersection of allowed documents.<br>
+         *
+         * @param limitedByIndexAccessControl {@link IndexAccessControl}
+         * @return {@link IndexAccessControl}
+         * @see FieldPermissions#limitFieldPermissions(FieldPermissions)
+         * @see DocumentPermissions#limitDocumentPermissions(DocumentPermissions)
+         */
+        public IndexAccessControl limitIndexAccessControl(IndexAccessControl limitedByIndexAccessControl) {
+            final boolean granted;
+            if (this.granted == limitedByIndexAccessControl.granted) {
+                granted = this.granted;
+            } else {
+                granted = false;
+            }
+            FieldPermissions fieldPermissions = getFieldPermissions().limitFieldPermissions(
+                    limitedByIndexAccessControl.fieldPermissions);
+            DocumentPermissions documentPermissions = getDocumentPermissions()
+                    .limitDocumentPermissions(limitedByIndexAccessControl.getDocumentPermissions());
+            return new IndexAccessControl(granted, fieldPermissions, documentPermissions);
         }
 
         @Override
@@ -91,11 +118,38 @@ public class IndicesAccessControl {
             return "IndexAccessControl{" +
                     "granted=" + granted +
                     ", fieldPermissions=" + fieldPermissions +
-                    ", queries=" + queries +
+                    ", documentPermissions=" + documentPermissions +
                     '}';
         }
     }
 
+    /**
+     * Returns a instance of {@link IndicesAccessControl}, where the privileges for {@code this}
+     * object are constrained by the privileges contained in the provided parameter.<br>
+     *
+     * @param limitedByIndicesAccessControl {@link IndicesAccessControl}
+     * @return {@link IndicesAccessControl}
+     */
+    public IndicesAccessControl limitIndicesAccessControl(IndicesAccessControl limitedByIndicesAccessControl) {
+        final boolean granted;
+        if (this.granted == limitedByIndicesAccessControl.granted) {
+            granted = this.granted;
+        } else {
+            granted = false;
+        }
+        Set<String> indexes = indexPermissions.keySet();
+        Set<String> otherIndexes = limitedByIndicesAccessControl.indexPermissions.keySet();
+        Set<String> commonIndexes = Sets.intersection(indexes, otherIndexes);
+
+        Map<String, IndexAccessControl> indexPermissions = new HashMap<>(commonIndexes.size());
+        for (String index : commonIndexes) {
+            IndexAccessControl indexAccessControl = getIndexPermissions(index);
+            IndexAccessControl limitedByIndexAccessControl = limitedByIndicesAccessControl.getIndexPermissions(index);
+            indexPermissions.put(index, indexAccessControl.limitIndexAccessControl(limitedByIndexAccessControl));
+        }
+        return new IndicesAccessControl(granted, indexPermissions);
+    }
+
     @Override
     public String toString() {
         return "IndicesAccessControl{" +

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapper.java

@@ -5,8 +5,8 @@
  */
 package org.elasticsearch.xpack.core.security.authz.accesscontrol;
 
-import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.BooleanQuery;
@@ -18,64 +18,35 @@ import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.LeafCollector;
-import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.Weight;
-import org.apache.lucene.search.join.BitSetProducer;
-import org.apache.lucene.search.join.ToChildBlockJoinQuery;
 import org.apache.lucene.util.BitSet;
 import org.apache.lucene.util.BitSetIterator;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.SparseFixedBitSet;
-import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.logging.LoggerMessageFormat;
-import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
-import org.elasticsearch.common.xcontent.NamedXContentRegistry;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 import org.elasticsearch.index.engine.EngineException;
-import org.elasticsearch.index.query.BoolQueryBuilder;
-import org.elasticsearch.index.query.BoostingQueryBuilder;
-import org.elasticsearch.index.query.ConstantScoreQueryBuilder;
-import org.elasticsearch.index.query.GeoShapeQueryBuilder;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.index.query.QueryRewriteContext;
 import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.index.query.Rewriteable;
-import org.elasticsearch.index.query.TermsQueryBuilder;
-import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
-import org.elasticsearch.index.search.NestedHelper;
 import org.elasticsearch.index.shard.IndexSearcherWrapper;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardUtils;
 import org.elasticsearch.license.XPackLicenseState;
-import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.script.ScriptType;
-import org.elasticsearch.script.TemplateScript;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField;
 import org.elasticsearch.xpack.core.security.authz.accesscontrol.DocumentSubsetReader.DocumentSubsetDirectoryReader;
+import org.elasticsearch.xpack.core.security.authz.permission.DocumentPermissions;
 import org.elasticsearch.xpack.core.security.support.Exceptions;
 import org.elasticsearch.xpack.core.security.user.User;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.function.Function;
 
-import static org.apache.lucene.search.BooleanClause.Occur.FILTER;
-import static org.apache.lucene.search.BooleanClause.Occur.SHOULD;
-
 /**
  * An {@link IndexSearcherWrapper} implementation that is used for field and document level security.
  * <p>
@@ -107,7 +78,7 @@ public class SecurityIndexSearcherWrapper extends IndexSearcherWrapper {
     }
 
     @Override
-    protected DirectoryReader wrap(DirectoryReader reader) {
+    protected DirectoryReader wrap(final DirectoryReader reader) {
         if (licenseState.isDocumentAndFieldLevelSecurityAllowed() == false) {
             return reader;
         }
@@ -120,47 +91,22 @@ public class SecurityIndexSearcherWrapper extends IndexSearcherWrapper {
                 throw new IllegalStateException(LoggerMessageFormat.format("couldn't extract shardId from reader [{}]", reader));
             }
 
-            IndicesAccessControl.IndexAccessControl permissions = indicesAccessControl.getIndexPermissions(shardId.getIndexName());
+            final IndicesAccessControl.IndexAccessControl permissions = indicesAccessControl.getIndexPermissions(shardId.getIndexName());
             // No permissions have been defined for an index, so don't intercept the index reader for access control
             if (permissions == null) {
                 return reader;
             }
 
-            if (permissions.getQueries() != null) {
-                BooleanQuery.Builder filter = new BooleanQuery.Builder();
-                for (BytesReference bytesReference : permissions.getQueries()) {
-                    QueryShardContext queryShardContext = queryShardContextProvider.apply(shardId);
-                    String templateResult = evaluateTemplate(bytesReference.utf8ToString());
-                    try (XContentParser parser = XContentFactory.xContent(templateResult)
-                            .createParser(queryShardContext.getXContentRegistry(), LoggingDeprecationHandler.INSTANCE, templateResult)) {
-                        QueryBuilder queryBuilder = queryShardContext.parseInnerQueryBuilder(parser);
-                        verifyRoleQuery(queryBuilder);
-                        failIfQueryUsesClient(queryBuilder, queryShardContext);
-                        Query roleQuery = queryShardContext.toQuery(queryBuilder).query();
-                        filter.add(roleQuery, SHOULD);
-                        if (queryShardContext.getMapperService().hasNested()) {
-                            NestedHelper nestedHelper = new NestedHelper(queryShardContext.getMapperService());
-                            if (nestedHelper.mightMatchNestedDocs(roleQuery)) {
-                                roleQuery = new BooleanQuery.Builder()
-                                    .add(roleQuery, FILTER)
-                                    .add(Queries.newNonNestedFilter(queryShardContext.indexVersionCreated()), FILTER)
-                                    .build();
-                            }
-                            // If access is allowed on root doc then also access is allowed on all nested docs of that root document:
-                            BitSetProducer rootDocs = queryShardContext.bitsetFilter(
-                                    Queries.newNonNestedFilter(queryShardContext.indexVersionCreated()));
-                            ToChildBlockJoinQuery includeNestedDocs = new ToChildBlockJoinQuery(roleQuery, rootDocs);
-                            filter.add(includeNestedDocs, SHOULD);
-                        }
-                    }
+            DirectoryReader wrappedReader = reader;
+            DocumentPermissions documentPermissions = permissions.getDocumentPermissions();
+            if (documentPermissions != null && documentPermissions.hasDocumentLevelPermissions()) {
+                BooleanQuery filterQuery = documentPermissions.filter(getUser(), scriptService, shardId, queryShardContextProvider);
+                if (filterQuery != null) {
+                    wrappedReader = DocumentSubsetReader.wrap(wrappedReader, bitsetFilterCache, new ConstantScoreQuery(filterQuery));
                 }
-
-                // at least one of the queries should match
-                filter.setMinimumNumberShouldMatch(1);
-                reader = DocumentSubsetReader.wrap(reader, bitsetFilterCache, new ConstantScoreQuery(filter.build()));
             }
 
-            return permissions.getFieldPermissions().filter(reader);
+            return permissions.getFieldPermissions().filter(wrappedReader);
         } catch (IOException e) {
             logger.error("Unable to apply field level security");
             throw ExceptionsHelper.convertToElastic(e);
@@ -255,48 +201,6 @@ public class SecurityIndexSearcherWrapper extends IndexSearcherWrapper {
         }
     }
 
-    String evaluateTemplate(String querySource) throws IOException {
-        // EMPTY is safe here because we never use namedObject
-        try (XContentParser parser = XContentFactory.xContent(querySource).createParser(NamedXContentRegistry.EMPTY,
-                LoggingDeprecationHandler.INSTANCE, querySource)) {
-            XContentParser.Token token = parser.nextToken();
-            if (token != XContentParser.Token.START_OBJECT) {
-                throw new ElasticsearchParseException("Unexpected token [" + token + "]");
-            }
-            token = parser.nextToken();
-            if (token != XContentParser.Token.FIELD_NAME) {
-                throw new ElasticsearchParseException("Unexpected token [" + token + "]");
-            }
-            if ("template".equals(parser.currentName())) {
-                token = parser.nextToken();
-                if (token != XContentParser.Token.START_OBJECT) {
-                    throw new ElasticsearchParseException("Unexpected token [" + token + "]");
-                }
-                Script script = Script.parse(parser);
-                // Add the user details to the params
-                Map<String, Object> params = new HashMap<>();
-                if (script.getParams() != null) {
-                    params.putAll(script.getParams());
-                }
-                User user = getUser();
-                Map<String, Object> userModel = new HashMap<>();
-                userModel.put("username", user.principal());
-                userModel.put("full_name", user.fullName());
-                userModel.put("email", user.email());
-                userModel.put("roles", Arrays.asList(user.roles()));
-                userModel.put("metadata", Collections.unmodifiableMap(user.metadata()));
-                params.put("_user", userModel);
-                // Always enforce mustache script lang:
-                script = new Script(script.getType(),
-                        script.getType() == ScriptType.STORED ? null : "mustache", script.getIdOrCode(), script.getOptions(), params);
-                TemplateScript compiledTemplate = scriptService.compile(script, TemplateScript.CONTEXT).newInstance(script.getParams());
-                return compiledTemplate.execute();
-            } else {
-                return querySource;
-            }
-        }
-    }
-
     protected IndicesAccessControl getIndicesAccessControl() {
         IndicesAccessControl indicesAccessControl = threadContext.getTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY);
         if (indicesAccessControl == null) {
@@ -310,65 +214,4 @@ public class SecurityIndexSearcherWrapper extends IndexSearcherWrapper {
         return authentication.getUser();
     }
 
-    /**
-     * Checks whether the role query contains queries we know can't be used as DLS role query.
-     */
-    static void verifyRoleQuery(QueryBuilder queryBuilder) throws IOException {
-        if (queryBuilder instanceof TermsQueryBuilder) {
-            TermsQueryBuilder termsQueryBuilder = (TermsQueryBuilder) queryBuilder;
-            if (termsQueryBuilder.termsLookup() != null) {
-                throw new IllegalArgumentException("terms query with terms lookup isn't supported as part of a role query");
-            }
-        } else if (queryBuilder instanceof GeoShapeQueryBuilder) {
-            GeoShapeQueryBuilder geoShapeQueryBuilder = (GeoShapeQueryBuilder) queryBuilder;
-            if (geoShapeQueryBuilder.shape() == null) {
-                throw new IllegalArgumentException("geoshape query referring to indexed shapes isn't support as part of a role query");
-            }
-        } else if (queryBuilder.getName().equals("percolate")) {
-            // actually only if percolate query is referring to an existing document then this is problematic,
-            // a normal percolate query does work. However we can't check that here as this query builder is inside
-            // another module. So we don't allow the entire percolate query. I don't think users would ever use
-            // a percolate query as role query, so this restriction shouldn't prohibit anyone from using dls.
-            throw new IllegalArgumentException("percolate query isn't support as part of a role query");
-        } else if (queryBuilder.getName().equals("has_child")) {
-            throw new IllegalArgumentException("has_child query isn't support as part of a role query");
-        } else if (queryBuilder.getName().equals("has_parent")) {
-            throw new IllegalArgumentException("has_parent query isn't support as part of a role query");
-        } else if (queryBuilder instanceof BoolQueryBuilder) {
-            BoolQueryBuilder boolQueryBuilder = (BoolQueryBuilder) queryBuilder;
-            List<QueryBuilder> clauses = new ArrayList<>();
-            clauses.addAll(boolQueryBuilder.filter());
-            clauses.addAll(boolQueryBuilder.must());
-            clauses.addAll(boolQueryBuilder.mustNot());
-            clauses.addAll(boolQueryBuilder.should());
-            for (QueryBuilder clause : clauses) {
-                verifyRoleQuery(clause);
-            }
-        } else if (queryBuilder instanceof ConstantScoreQueryBuilder) {
-            verifyRoleQuery(((ConstantScoreQueryBuilder) queryBuilder).innerQuery());
-        } else if (queryBuilder instanceof FunctionScoreQueryBuilder) {
-            verifyRoleQuery(((FunctionScoreQueryBuilder) queryBuilder).query());
-        } else if (queryBuilder instanceof BoostingQueryBuilder) {
-            verifyRoleQuery(((BoostingQueryBuilder) queryBuilder).negativeQuery());
-            verifyRoleQuery(((BoostingQueryBuilder) queryBuilder).positiveQuery());
-        }
-    }
-
-    /**
-     * Fall back validation that verifies that queries during rewrite don't use
-     * the client to make remote calls. In the case of DLS this can cause a dead
-     * lock if DLS is also applied on these remote calls. For example in the
-     * case of terms query with lookup, this can cause recursive execution of
-     * the DLS query until the get thread pool has been exhausted:
-     * https://github.com/elastic/x-plugins/issues/3145
-     */
-    static void failIfQueryUsesClient(QueryBuilder queryBuilder, QueryRewriteContext original)
-            throws IOException {
-        QueryRewriteContext copy = new QueryRewriteContext(
-                original.getXContentRegistry(), original.getWriteableRegistry(), null, original::nowInMillis);
-        Rewriteable.rewrite(queryBuilder, copy);
-        if (copy.hasAsyncActions()) {
-            throw new IllegalStateException("role queries are not allowed to execute additional requests");
-        }
-    }
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ApplicationPermission.java

@@ -12,10 +12,12 @@ import org.apache.lucene.util.automaton.Operations;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
 import org.elasticsearch.xpack.core.security.support.Automatons;
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -83,6 +85,40 @@ public final class ApplicationPermission {
         return matched;
     }
 
+    /**
+     * For a given application, checks for the privileges for resources and returns an instance of {@link ResourcePrivilegesMap} holding a
+     * map of resource to {@link ResourcePrivileges} where the resource is application resource and the map of application privilege to
+     * whether it is allowed or not.
+     *
+     * @param applicationName checks privileges for the provided application name
+     * @param checkForResources check permission grants for the set of resources
+     * @param checkForPrivilegeNames check permission grants for the set of privilege names
+     * @param storedPrivileges stored {@link ApplicationPrivilegeDescriptor} for an application against which the access checks are
+     *        performed
+     * @return an instance of {@link ResourcePrivilegesMap}
+     */
+    public ResourcePrivilegesMap checkResourcePrivileges(final String applicationName, Set<String> checkForResources,
+                                                         Set<String> checkForPrivilegeNames,
+                                                         Collection<ApplicationPrivilegeDescriptor> storedPrivileges) {
+        final ResourcePrivilegesMap.Builder resourcePrivilegesMapBuilder = ResourcePrivilegesMap.builder();
+        for (String checkResource : checkForResources) {
+            for (String checkPrivilegeName : checkForPrivilegeNames) {
+                final Set<String> nameSet = Collections.singleton(checkPrivilegeName);
+                final ApplicationPrivilege checkPrivilege = ApplicationPrivilege.get(applicationName, nameSet, storedPrivileges);
+                assert checkPrivilege.getApplication().equals(applicationName) : "Privilege " + checkPrivilege + " should have application "
+                        + applicationName;
+                assert checkPrivilege.name().equals(nameSet) : "Privilege " + checkPrivilege + " should have name " + nameSet;
+
+                if (grants(checkPrivilege, checkResource)) {
+                    resourcePrivilegesMapBuilder.addResourcePrivilege(checkResource, checkPrivilegeName, Boolean.TRUE);
+                } else {
+                    resourcePrivilegesMapBuilder.addResourcePrivilege(checkResource, checkPrivilegeName, Boolean.FALSE);
+                }
+            }
+        }
+        return resourcePrivilegesMapBuilder.build();
+    }
+
     @Override
     public String toString() {
         return getClass().getSimpleName() + "{privileges=" + permissions + "}";

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ClusterPermission.java

@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.core.security.authz.permission;
 
+import org.apache.lucene.util.automaton.Operations;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilege;
@@ -33,6 +34,10 @@ public abstract class ClusterPermission {
 
     public abstract boolean check(String action, TransportRequest request);
 
+    public boolean grants(ClusterPrivilege clusterPrivilege) {
+        return Operations.subsetOf(clusterPrivilege.getAutomaton(), this.privilege().getAutomaton());
+    }
+
     public abstract List<Tuple<ClusterPrivilege, ConditionalClusterPrivilege>> privileges();
 
     /**
@@ -111,5 +116,10 @@ public abstract class ClusterPermission {
         public boolean check(String action, TransportRequest request) {
             return children.stream().anyMatch(p -> p.check(action, request));
         }
+
+        @Override
+        public boolean grants(ClusterPrivilege clusterPrivilege) {
+            return children.stream().anyMatch(p -> p.grants(clusterPrivilege));
+        }
     }
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/DocumentPermissions.java

@@ -0,0 +1,262 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.authz.permission;
+
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.join.BitSetProducer;
+import org.apache.lucene.search.join.ToChildBlockJoinQuery;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.lucene.search.Queries;
+import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.BoostingQueryBuilder;
+import org.elasticsearch.index.query.ConstantScoreQueryBuilder;
+import org.elasticsearch.index.query.GeoShapeQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryRewriteContext;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.query.Rewriteable;
+import org.elasticsearch.index.query.TermsQueryBuilder;
+import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
+import org.elasticsearch.index.search.NestedHelper;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.xpack.core.security.authz.support.SecurityQueryTemplateEvaluator;
+import org.elasticsearch.xpack.core.security.user.User;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.function.Function;
+
+import static org.apache.lucene.search.BooleanClause.Occur.FILTER;
+import static org.apache.lucene.search.BooleanClause.Occur.SHOULD;
+
+/**
+ * Stores document level permissions in the form queries that match all the accessible documents.<br>
+ * The document level permissions may be limited by another set of queries in that case the limited
+ * queries are used as an additional filter.
+ */
+public final class DocumentPermissions {
+    private final Set<BytesReference> queries;
+    private final Set<BytesReference> limitedByQueries;
+
+    private static DocumentPermissions ALLOW_ALL = new DocumentPermissions();
+
+    DocumentPermissions() {
+        this.queries = null;
+        this.limitedByQueries = null;
+    }
+
+    DocumentPermissions(Set<BytesReference> queries) {
+        this(queries, null);
+    }
+
+    DocumentPermissions(Set<BytesReference> queries, Set<BytesReference> scopedByQueries) {
+        if (queries == null && scopedByQueries == null) {
+            throw new IllegalArgumentException("one of the queries or scoped queries must be provided");
+        }
+        this.queries = (queries != null) ? Collections.unmodifiableSet(queries) : queries;
+        this.limitedByQueries = (scopedByQueries != null) ? Collections.unmodifiableSet(scopedByQueries) : scopedByQueries;
+    }
+
+    public Set<BytesReference> getQueries() {
+        return queries;
+    }
+
+    public Set<BytesReference> getLimitedByQueries() {
+        return limitedByQueries;
+    }
+
+    /**
+     * @return {@code true} if either queries or scoped queries are present for document level security else returns {@code false}
+     */
+    public boolean hasDocumentLevelPermissions() {
+        return queries != null || limitedByQueries != null;
+    }
+
+    /**
+     * Creates a {@link BooleanQuery} to be used as filter to restrict access to documents.<br>
+     * Document permission queries are used to create an boolean query.<br>
+     * If the document permissions are limited, then there is an additional filter added restricting access to documents only allowed by the
+     * limited queries.
+     *
+     * @param user authenticated {@link User}
+     * @param scriptService {@link ScriptService} for evaluating query templates
+     * @param shardId {@link ShardId}
+     * @param queryShardContextProvider {@link QueryShardContext}
+     * @return {@link BooleanQuery} for the filter
+     * @throws IOException thrown if there is an exception during parsing
+     */
+    public BooleanQuery filter(User user, ScriptService scriptService, ShardId shardId,
+                                      Function<ShardId, QueryShardContext> queryShardContextProvider) throws IOException {
+        if (hasDocumentLevelPermissions()) {
+            BooleanQuery.Builder filter;
+            if (queries != null && limitedByQueries != null) {
+                filter = new BooleanQuery.Builder();
+                BooleanQuery.Builder scopedFilter = new BooleanQuery.Builder();
+                buildRoleQuery(user, scriptService, shardId, queryShardContextProvider, limitedByQueries, scopedFilter);
+                filter.add(scopedFilter.build(), FILTER);
+
+                buildRoleQuery(user, scriptService, shardId, queryShardContextProvider, queries, filter);
+            } else if (queries != null) {
+                filter = new BooleanQuery.Builder();
+                buildRoleQuery(user, scriptService, shardId, queryShardContextProvider, queries, filter);
+            } else if (limitedByQueries != null) {
+                filter = new BooleanQuery.Builder();
+                buildRoleQuery(user, scriptService, shardId, queryShardContextProvider, limitedByQueries, filter);
+            } else {
+                return null;
+            }
+            return filter.build();
+        }
+        return null;
+    }
+
+    private static void buildRoleQuery(User user, ScriptService scriptService, ShardId shardId,
+                                       Function<ShardId, QueryShardContext> queryShardContextProvider, Set<BytesReference> queries,
+                                       BooleanQuery.Builder filter) throws IOException {
+        for (BytesReference bytesReference : queries) {
+            QueryShardContext queryShardContext = queryShardContextProvider.apply(shardId);
+            String templateResult = SecurityQueryTemplateEvaluator.evaluateTemplate(bytesReference.utf8ToString(), scriptService, user);
+            try (XContentParser parser = XContentFactory.xContent(templateResult).createParser(queryShardContext.getXContentRegistry(),
+                    LoggingDeprecationHandler.INSTANCE, templateResult)) {
+                QueryBuilder queryBuilder = queryShardContext.parseInnerQueryBuilder(parser);
+                verifyRoleQuery(queryBuilder);
+                failIfQueryUsesClient(queryBuilder, queryShardContext);
+                Query roleQuery = queryShardContext.toQuery(queryBuilder).query();
+                filter.add(roleQuery, SHOULD);
+                if (queryShardContext.getMapperService().hasNested()) {
+                    NestedHelper nestedHelper = new NestedHelper(queryShardContext.getMapperService());
+                    if (nestedHelper.mightMatchNestedDocs(roleQuery)) {
+                        roleQuery = new BooleanQuery.Builder().add(roleQuery, FILTER)
+                                .add(Queries.newNonNestedFilter(queryShardContext.indexVersionCreated()), FILTER).build();
+                    }
+                    // If access is allowed on root doc then also access is allowed on all nested docs of that root document:
+                    BitSetProducer rootDocs = queryShardContext
+                            .bitsetFilter(Queries.newNonNestedFilter(queryShardContext.indexVersionCreated()));
+                    ToChildBlockJoinQuery includeNestedDocs = new ToChildBlockJoinQuery(roleQuery, rootDocs);
+                    filter.add(includeNestedDocs, SHOULD);
+                }
+            }
+        }
+        // at least one of the queries should match
+        filter.setMinimumNumberShouldMatch(1);
+    }
+
+    /**
+     * Checks whether the role query contains queries we know can't be used as DLS role query.
+     */
+    static void verifyRoleQuery(QueryBuilder queryBuilder) throws IOException {
+        if (queryBuilder instanceof TermsQueryBuilder) {
+            TermsQueryBuilder termsQueryBuilder = (TermsQueryBuilder) queryBuilder;
+            if (termsQueryBuilder.termsLookup() != null) {
+                throw new IllegalArgumentException("terms query with terms lookup isn't supported as part of a role query");
+            }
+        } else if (queryBuilder instanceof GeoShapeQueryBuilder) {
+            GeoShapeQueryBuilder geoShapeQueryBuilder = (GeoShapeQueryBuilder) queryBuilder;
+            if (geoShapeQueryBuilder.shape() == null) {
+                throw new IllegalArgumentException("geoshape query referring to indexed shapes isn't support as part of a role query");
+            }
+        } else if (queryBuilder.getName().equals("percolate")) {
+            // actually only if percolate query is referring to an existing document then this is problematic,
+            // a normal percolate query does work. However we can't check that here as this query builder is inside
+            // another module. So we don't allow the entire percolate query. I don't think users would ever use
+            // a percolate query as role query, so this restriction shouldn't prohibit anyone from using dls.
+            throw new IllegalArgumentException("percolate query isn't support as part of a role query");
+        } else if (queryBuilder.getName().equals("has_child")) {
+            throw new IllegalArgumentException("has_child query isn't support as part of a role query");
+        } else if (queryBuilder.getName().equals("has_parent")) {
+            throw new IllegalArgumentException("has_parent query isn't support as part of a role query");
+        } else if (queryBuilder instanceof BoolQueryBuilder) {
+            BoolQueryBuilder boolQueryBuilder = (BoolQueryBuilder) queryBuilder;
+            List<QueryBuilder> clauses = new ArrayList<>();
+            clauses.addAll(boolQueryBuilder.filter());
+            clauses.addAll(boolQueryBuilder.must());
+            clauses.addAll(boolQueryBuilder.mustNot());
+            clauses.addAll(boolQueryBuilder.should());
+            for (QueryBuilder clause : clauses) {
+                verifyRoleQuery(clause);
+            }
+        } else if (queryBuilder instanceof ConstantScoreQueryBuilder) {
+            verifyRoleQuery(((ConstantScoreQueryBuilder) queryBuilder).innerQuery());
+        } else if (queryBuilder instanceof FunctionScoreQueryBuilder) {
+            verifyRoleQuery(((FunctionScoreQueryBuilder) queryBuilder).query());
+        } else if (queryBuilder instanceof BoostingQueryBuilder) {
+            verifyRoleQuery(((BoostingQueryBuilder) queryBuilder).negativeQuery());
+            verifyRoleQuery(((BoostingQueryBuilder) queryBuilder).positiveQuery());
+        }
+    }
+
+    /**
+     * Fall back validation that verifies that queries during rewrite don't use
+     * the client to make remote calls. In the case of DLS this can cause a dead
+     * lock if DLS is also applied on these remote calls. For example in the
+     * case of terms query with lookup, this can cause recursive execution of
+     * the DLS query until the get thread pool has been exhausted:
+     * https://github.com/elastic/x-plugins/issues/3145
+     */
+    static void failIfQueryUsesClient(QueryBuilder queryBuilder, QueryRewriteContext original)
+            throws IOException {
+        QueryRewriteContext copy = new QueryRewriteContext(
+                original.getXContentRegistry(), original.getWriteableRegistry(), null, original::nowInMillis);
+        Rewriteable.rewrite(queryBuilder, copy);
+        if (copy.hasAsyncActions()) {
+            throw new IllegalStateException("role queries are not allowed to execute additional requests");
+        }
+    }
+
+    /**
+     * Create {@link DocumentPermissions} for given set of queries
+     * @param queries set of queries
+     * @return {@link DocumentPermissions}
+     */
+    public static DocumentPermissions filteredBy(Set<BytesReference> queries) {
+        if (queries == null || queries.isEmpty()) {
+            throw new IllegalArgumentException("null or empty queries not permitted");
+        }
+        return new DocumentPermissions(queries);
+    }
+
+    /**
+     * Create {@link DocumentPermissions} with no restriction. The {@link #getQueries()}
+     * will return {@code null} in this case and {@link #hasDocumentLevelPermissions()}
+     * will be {@code false}
+     * @return {@link DocumentPermissions}
+     */
+    public static DocumentPermissions allowAll() {
+        return ALLOW_ALL;
+    }
+
+    /**
+     * Create a document permissions, where the permissions for {@code this} are
+     * limited by the queries from other document permissions.<br>
+     *
+     * @param limitedByDocumentPermissions {@link DocumentPermissions} used to limit the document level access
+     * @return instance of {@link DocumentPermissions}
+     */
+    public DocumentPermissions limitDocumentPermissions(
+            DocumentPermissions limitedByDocumentPermissions) {
+        assert limitedByQueries == null
+                && limitedByDocumentPermissions.limitedByQueries == null : "nested scoping for document permissions is not permitted";
+        if (queries == null && limitedByDocumentPermissions.queries == null) {
+            return DocumentPermissions.allowAll();
+        }
+        return new DocumentPermissions(queries, limitedByDocumentPermissions.queries);
+    }
+
+    @Override
+    public String toString() {
+        return "DocumentPermissions [queries=" + queries + ", scopedByQueries=" + limitedByQueries + "]";
+    }
+
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java

@@ -90,13 +90,15 @@ public final class FieldPermissions implements Accountable {
 
         long ramBytesUsed = BASE_FIELD_PERM_DEF_BYTES;
 
-        for (FieldGrantExcludeGroup group : fieldPermissionsDefinition.getFieldGrantExcludeGroups()) {
-            ramBytesUsed += BASE_FIELD_GROUP_BYTES + BASE_HASHSET_ENTRY_SIZE;
-            if (group.getGrantedFields() != null) {
-                ramBytesUsed += RamUsageEstimator.shallowSizeOf(group.getGrantedFields());
-            }
-            if (group.getExcludedFields() != null) {
-                ramBytesUsed += RamUsageEstimator.shallowSizeOf(group.getExcludedFields());
+        if (fieldPermissionsDefinition != null) {
+            for (FieldGrantExcludeGroup group : fieldPermissionsDefinition.getFieldGrantExcludeGroups()) {
+                ramBytesUsed += BASE_FIELD_GROUP_BYTES + BASE_HASHSET_ENTRY_SIZE;
+                if (group.getGrantedFields() != null) {
+                    ramBytesUsed += RamUsageEstimator.shallowSizeOf(group.getGrantedFields());
+                }
+                if (group.getExcludedFields() != null) {
+                    ramBytesUsed += RamUsageEstimator.shallowSizeOf(group.getExcludedFields());
+                }
             }
         }
         ramBytesUsed += permittedFieldsAutomaton.ramBytesUsed();
@@ -153,6 +155,28 @@ public final class FieldPermissions implements Accountable {
         return grantedFieldsAutomaton;
     }
 
+    /**
+     * Returns a field permissions instance where it is limited by the given field permissions.<br>
+     * If the current and the other field permissions have field level security then it takes
+     * an intersection of permitted fields.<br>
+     * If none of the permissions have field level security enabled, then returns permissions
+     * instance where all fields are allowed.
+     *
+     * @param limitedBy {@link FieldPermissions} used to limit current field permissions
+     * @return {@link FieldPermissions}
+     */
+    public FieldPermissions limitFieldPermissions(FieldPermissions limitedBy) {
+        if (hasFieldLevelSecurity() && limitedBy != null && limitedBy.hasFieldLevelSecurity()) {
+            Automaton permittedFieldsAutomaton = Automatons.intersectAndMinimize(getIncludeAutomaton(), limitedBy.getIncludeAutomaton());
+            return new FieldPermissions(null, permittedFieldsAutomaton);
+        } else if (limitedBy != null && limitedBy.hasFieldLevelSecurity()) {
+            return new FieldPermissions(limitedBy.getFieldPermissionsDefinition(), limitedBy.getIncludeAutomaton());
+        } else if (hasFieldLevelSecurity()) {
+            return new FieldPermissions(getFieldPermissionsDefinition(), getIncludeAutomaton());
+        }
+        return FieldPermissions.DEFAULT;
+    }
+
     /**
      * Returns true if this field permission policy allows access to the field and false if not.
      * fieldName can be a wildcard.
@@ -178,7 +202,6 @@ public final class FieldPermissions implements Accountable {
         return FieldSubsetReader.wrap(reader, permittedFieldsAutomaton);
     }
 
-    // for testing only
     Automaton getIncludeAutomaton() {
         return originalAutomaton;
     }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java

@@ -7,6 +7,7 @@ package org.elasticsearch.xpack.core.security.authz.permission;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.lucene.util.automaton.Automaton;
+import org.apache.lucene.util.automaton.Operations;
 import org.apache.lucene.util.automaton.TooComplexToDeterminizeException;
 import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.cluster.metadata.AliasOrIndex;
@@ -23,6 +24,7 @@ import org.elasticsearch.xpack.core.security.support.Automatons;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -123,6 +125,49 @@ public final class IndicesPermission {
         return false;
     }
 
+    /**
+     * For given index patterns and index privileges determines allowed privileges and creates an instance of {@link ResourcePrivilegesMap}
+     * holding a map of resource to {@link ResourcePrivileges} where resource is index pattern and the map of index privilege to whether it
+     * is allowed or not.
+     *
+     * @param checkForIndexPatterns check permission grants for the set of index patterns
+     * @param allowRestrictedIndices if {@code true} then checks permission grants even for restricted indices by index matching
+     * @param checkForPrivileges check permission grants for the set of index privileges
+     * @return an instance of {@link ResourcePrivilegesMap}
+     */
+    public ResourcePrivilegesMap checkResourcePrivileges(Set<String> checkForIndexPatterns, boolean allowRestrictedIndices,
+                                                         Set<String> checkForPrivileges) {
+        final ResourcePrivilegesMap.Builder resourcePrivilegesMapBuilder = ResourcePrivilegesMap.builder();
+        final Map<IndicesPermission.Group, Automaton> predicateCache = new HashMap<>();
+        for (String forIndexPattern : checkForIndexPatterns) {
+            final Automaton checkIndexAutomaton = IndicesPermission.Group.buildIndexMatcherAutomaton(allowRestrictedIndices,
+                    forIndexPattern);
+            Automaton allowedIndexPrivilegesAutomaton = null;
+            for (Group group : groups) {
+                final Automaton groupIndexAutomaton = predicateCache.computeIfAbsent(group,
+                        g -> IndicesPermission.Group.buildIndexMatcherAutomaton(g.allowRestrictedIndices(), g.indices()));
+                if (Operations.subsetOf(checkIndexAutomaton, groupIndexAutomaton)) {
+                    if (allowedIndexPrivilegesAutomaton != null) {
+                        allowedIndexPrivilegesAutomaton = Automatons
+                                .unionAndMinimize(Arrays.asList(allowedIndexPrivilegesAutomaton, group.privilege().getAutomaton()));
+                    } else {
+                        allowedIndexPrivilegesAutomaton = group.privilege().getAutomaton();
+                    }
+                }
+            }
+            for (String privilege : checkForPrivileges) {
+                IndexPrivilege indexPrivilege = IndexPrivilege.get(Collections.singleton(privilege));
+                if (allowedIndexPrivilegesAutomaton != null
+                        && Operations.subsetOf(indexPrivilege.getAutomaton(), allowedIndexPrivilegesAutomaton)) {
+                    resourcePrivilegesMapBuilder.addResourcePrivilege(forIndexPattern, privilege, Boolean.TRUE);
+                } else {
+                    resourcePrivilegesMapBuilder.addResourcePrivilege(forIndexPattern, privilege, Boolean.FALSE);
+                }
+            }
+        }
+        return resourcePrivilegesMapBuilder.build();
+    }
+
     public Automaton allowedActionsMatcher(String index) {
         List<Automaton> automatonList = new ArrayList<>();
         for (Group group : groups) {
@@ -207,7 +252,8 @@ public final class IndicesPermission {
             } else {
                 fieldPermissions = FieldPermissions.DEFAULT;
             }
-            indexPermissions.put(index, new IndicesAccessControl.IndexAccessControl(entry.getValue(), fieldPermissions, roleQueries));
+            indexPermissions.put(index, new IndicesAccessControl.IndexAccessControl(entry.getValue(), fieldPermissions,
+                    (roleQueries != null) ? DocumentPermissions.filteredBy(roleQueries) : DocumentPermissions.allowAll()));
         }
         return unmodifiableMap(indexPermissions);
     }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/LimitedRole.java

@@ -0,0 +1,152 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.authz.permission;
+
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.transport.TransportRequest;
+import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
+import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilege;
+
+import java.util.Collection;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.Predicate;
+
+/**
+ * A {@link Role} limited by another role.<br>
+ * The effective permissions returned on {@link #authorize(String, Set, MetaData, FieldPermissionsCache)} call would be limited by the
+ * provided role.
+ */
+public final class LimitedRole extends Role {
+    private final Role limitedBy;
+
+    LimitedRole(String[] names, ClusterPermission cluster, IndicesPermission indices, ApplicationPermission application,
+            RunAsPermission runAs, Role limitedBy) {
+        super(names, cluster, indices, application, runAs);
+        assert limitedBy != null : "limiting role is required";
+        this.limitedBy = limitedBy;
+    }
+
+    public Role limitedBy() {
+        return limitedBy;
+    }
+
+    @Override
+    public IndicesAccessControl authorize(String action, Set<String> requestedIndicesOrAliases, MetaData metaData,
+                                          FieldPermissionsCache fieldPermissionsCache) {
+        IndicesAccessControl indicesAccessControl = super.authorize(action, requestedIndicesOrAliases, metaData, fieldPermissionsCache);
+        IndicesAccessControl limitedByIndicesAccessControl = limitedBy.authorize(action, requestedIndicesOrAliases, metaData,
+                fieldPermissionsCache);
+
+        return indicesAccessControl.limitIndicesAccessControl(limitedByIndicesAccessControl);
+    }
+
+    /**
+     * @return A predicate that will match all the indices that this role and the limited by role has the privilege for executing the given
+     * action on.
+     */
+    @Override
+    public Predicate<String> allowedIndicesMatcher(String action) {
+        Predicate<String> predicate = indices().allowedIndicesMatcher(action);
+        predicate = predicate.and(limitedBy.indices().allowedIndicesMatcher(action));
+        return predicate;
+    }
+
+    /**
+     * Check if indices permissions allow for the given action, also checks whether the limited by role allows the given actions
+     *
+     * @param action indices action
+     * @return {@code true} if action is allowed else returns {@code false}
+     */
+    @Override
+    public boolean checkIndicesAction(String action) {
+        return super.checkIndicesAction(action) && limitedBy.checkIndicesAction(action);
+    }
+
+    /**
+     * For given index patterns and index privileges determines allowed privileges and creates an instance of {@link ResourcePrivilegesMap}
+     * holding a map of resource to {@link ResourcePrivileges} where resource is index pattern and the map of index privilege to whether it
+     * is allowed or not.<br>
+     * This one takes intersection of resource privileges with the resource privileges from the limited-by role.
+     *
+     * @param checkForIndexPatterns check permission grants for the set of index patterns
+     * @param allowRestrictedIndices if {@code true} then checks permission grants even for restricted indices by index matching
+     * @param checkForPrivileges check permission grants for the set of index privileges
+     * @return an instance of {@link ResourcePrivilegesMap}
+     */
+    @Override
+    public ResourcePrivilegesMap checkIndicesPrivileges(Set<String> checkForIndexPatterns, boolean allowRestrictedIndices,
+                                                        Set<String> checkForPrivileges) {
+        ResourcePrivilegesMap resourcePrivilegesMap = super.indices().checkResourcePrivileges(checkForIndexPatterns, allowRestrictedIndices,
+                checkForPrivileges);
+        ResourcePrivilegesMap resourcePrivilegesMapForLimitedRole = limitedBy.indices().checkResourcePrivileges(checkForIndexPatterns,
+                allowRestrictedIndices, checkForPrivileges);
+        return ResourcePrivilegesMap.intersection(resourcePrivilegesMap, resourcePrivilegesMapForLimitedRole);
+    }
+
+    /**
+     * Check if cluster permissions allow for the given action, also checks whether the limited by role allows the given actions
+     *
+     * @param action cluster action
+     * @param request {@link TransportRequest}
+     * @return {@code true} if action is allowed else returns {@code false}
+     */
+    @Override
+    public boolean checkClusterAction(String action, TransportRequest request) {
+        return super.checkClusterAction(action, request) && limitedBy.checkClusterAction(action, request);
+    }
+
+    /**
+     * Check if cluster permissions grants the given cluster privilege, also checks whether the limited by role grants the given cluster
+     * privilege
+     *
+     * @param clusterPrivilege cluster privilege
+     * @return {@code true} if cluster privilege is allowed else returns {@code false}
+     */
+    @Override
+    public boolean grants(ClusterPrivilege clusterPrivilege) {
+        return super.grants(clusterPrivilege) && limitedBy.grants(clusterPrivilege);
+    }
+
+    /**
+     * For a given application, checks for the privileges for resources and returns an instance of {@link ResourcePrivilegesMap} holding a
+     * map of resource to {@link ResourcePrivileges} where the resource is application resource and the map of application privilege to
+     * whether it is allowed or not.<br>
+     * This one takes intersection of resource privileges with the resource privileges from the limited-by role.
+     *
+     * @param applicationName checks privileges for the provided application name
+     * @param checkForResources check permission grants for the set of resources
+     * @param checkForPrivilegeNames check permission grants for the set of privilege names
+     * @param storedPrivileges stored {@link ApplicationPrivilegeDescriptor} for an application against which the access checks are
+     * performed
+     * @return an instance of {@link ResourcePrivilegesMap}
+     */
+    @Override
+    public ResourcePrivilegesMap checkApplicationResourcePrivileges(final String applicationName, Set<String> checkForResources,
+                                                                    Set<String> checkForPrivilegeNames,
+                                                                    Collection<ApplicationPrivilegeDescriptor> storedPrivileges) {
+        ResourcePrivilegesMap resourcePrivilegesMap = super.application().checkResourcePrivileges(applicationName, checkForResources,
+                checkForPrivilegeNames, storedPrivileges);
+        ResourcePrivilegesMap resourcePrivilegesMapForLimitedRole = limitedBy.application().checkResourcePrivileges(applicationName,
+                checkForResources, checkForPrivilegeNames, storedPrivileges);
+        return ResourcePrivilegesMap.intersection(resourcePrivilegesMap, resourcePrivilegesMapForLimitedRole);
+    }
+
+    /**
+     * Create a new role defined by given role and the limited role.
+     *
+     * @param fromRole existing role {@link Role}
+     * @param limitedByRole restrict the newly formed role to the permissions defined by this limited {@link Role}
+     * @return {@link LimitedRole}
+     */
+    public static LimitedRole createLimitedRole(Role fromRole, Role limitedByRole) {
+        Objects.requireNonNull(limitedByRole, "limited by role is required to create limited role");
+        return new LimitedRole(fromRole.names(), fromRole.cluster(), fromRole.indices(), fromRole.application(), fromRole.runAs(),
+                limitedByRole);
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ResourcePrivileges.java

@@ -0,0 +1,93 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.authz.permission;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+
+/**
+ * A generic structure to encapsulate resource to privileges map.
+ */
+public final class ResourcePrivileges {
+
+    private final String resource;
+    private final Map<String, Boolean> privileges;
+
+    ResourcePrivileges(String resource, Map<String, Boolean> privileges) {
+        this.resource = Objects.requireNonNull(resource);
+        this.privileges = Collections.unmodifiableMap(privileges);
+    }
+
+    public String getResource() {
+        return resource;
+    }
+
+    public Map<String, Boolean> getPrivileges() {
+        return privileges;
+    }
+
+    public boolean isAllowed(String privilege) {
+        return Boolean.TRUE.equals(privileges.get(privilege));
+    }
+
+    @Override
+    public String toString() {
+        return getClass().getSimpleName() + "{" + "resource='" + resource + '\'' + ", privileges=" + privileges + '}';
+    }
+
+    @Override
+    public int hashCode() {
+        int result = resource.hashCode();
+        result = 31 * result + privileges.hashCode();
+        return result;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+
+        final ResourcePrivileges other = (ResourcePrivileges) o;
+        return this.resource.equals(other.resource) && this.privileges.equals(other.privileges);
+    }
+
+    public static Builder builder(String resource) {
+        return new Builder(resource);
+    }
+
+    public static final class Builder {
+        private final String resource;
+        private Map<String, Boolean> privileges = new HashMap<>();
+
+        private Builder(String resource) {
+            this.resource = resource;
+        }
+
+        public Builder addPrivilege(String privilege, Boolean allowed) {
+            this.privileges.compute(privilege, (k, v) -> ((v == null) ? allowed : v && allowed));
+            return this;
+        }
+
+        public Builder addPrivileges(Map<String, Boolean> privileges) {
+            for (Entry<String, Boolean> entry : privileges.entrySet()) {
+                addPrivilege(entry.getKey(), entry.getValue());
+            }
+            return this;
+        }
+
+        public ResourcePrivileges build() {
+            return new ResourcePrivileges(resource, privileges);
+        }
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ResourcePrivilegesMap.java

@@ -0,0 +1,121 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.authz.permission;
+
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+/**
+ * A generic structure to encapsulate resources to {@link ResourcePrivileges}. Also keeps track of whether the resource privileges allow
+ * permissions to all resources.
+ */
+public final class ResourcePrivilegesMap {
+
+    private final boolean allAllowed;
+    private final Map<String, ResourcePrivileges> resourceToResourcePrivileges;
+
+    public ResourcePrivilegesMap(boolean allAllowed, Map<String, ResourcePrivileges> resToResPriv) {
+        this.allAllowed = allAllowed;
+        this.resourceToResourcePrivileges = Collections.unmodifiableMap(Objects.requireNonNull(resToResPriv));
+    }
+
+    public boolean allAllowed() {
+        return allAllowed;
+    }
+
+    public Map<String, ResourcePrivileges> getResourceToResourcePrivileges() {
+        return resourceToResourcePrivileges;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(allAllowed, resourceToResourcePrivileges);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null) {
+            return false;
+        }
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+        final ResourcePrivilegesMap other = (ResourcePrivilegesMap) obj;
+        return allAllowed == other.allAllowed && Objects.equals(resourceToResourcePrivileges, other.resourceToResourcePrivileges);
+    }
+
+    @Override
+    public String toString() {
+        return "ResourcePrivilegesMap [allAllowed=" + allAllowed + ", resourceToResourcePrivileges=" + resourceToResourcePrivileges + "]";
+    }
+
+    public static final class Builder {
+        private boolean allowAll = true;
+        private Map<String, ResourcePrivileges.Builder> resourceToResourcePrivilegesBuilder = new LinkedHashMap<>();
+
+        public Builder addResourcePrivilege(String resource, String privilege, Boolean allowed) {
+            assert resource != null && privilege != null
+                    && allowed != null : "resource, privilege and permission(allowed or denied) are required";
+            ResourcePrivileges.Builder builder = resourceToResourcePrivilegesBuilder.computeIfAbsent(resource, ResourcePrivileges::builder);
+            builder.addPrivilege(privilege, allowed);
+            allowAll = allowAll && allowed;
+            return this;
+        }
+
+        public Builder addResourcePrivilege(String resource, Map<String, Boolean> privilegePermissions) {
+            assert resource != null && privilegePermissions != null : "resource, privilege permissions(allowed or denied) are required";
+            ResourcePrivileges.Builder builder = resourceToResourcePrivilegesBuilder.computeIfAbsent(resource, ResourcePrivileges::builder);
+            builder.addPrivileges(privilegePermissions);
+            allowAll = allowAll && privilegePermissions.values().stream().allMatch(b -> Boolean.TRUE.equals(b));
+            return this;
+        }
+
+        public Builder addResourcePrivilegesMap(ResourcePrivilegesMap resourcePrivilegesMap) {
+            resourcePrivilegesMap.getResourceToResourcePrivileges().entrySet().stream()
+                    .forEach(e -> this.addResourcePrivilege(e.getKey(), e.getValue().getPrivileges()));
+            return this;
+        }
+
+        public ResourcePrivilegesMap build() {
+            Map<String, ResourcePrivileges> result = resourceToResourcePrivilegesBuilder.entrySet().stream()
+                    .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue().build()));
+            return new ResourcePrivilegesMap(allowAll, result);
+        }
+    }
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    /**
+     * Takes an intersection of resource privileges and returns a new instance of {@link ResourcePrivilegesMap}. If one of the resource
+     * privileges map does not allow access to a resource then the resulting map would also not allow access.
+     *
+     * @param left an instance of {@link ResourcePrivilegesMap}
+     * @param right an instance of {@link ResourcePrivilegesMap}
+     * @return a new instance of {@link ResourcePrivilegesMap}, an intersection of resource privileges.
+     */
+    public static ResourcePrivilegesMap intersection(final ResourcePrivilegesMap left, final ResourcePrivilegesMap right) {
+        Objects.requireNonNull(left);
+        Objects.requireNonNull(right);
+        final ResourcePrivilegesMap.Builder builder = ResourcePrivilegesMap.builder();
+        for (Entry<String, ResourcePrivileges> leftResPrivsEntry : left.getResourceToResourcePrivileges().entrySet()) {
+            final ResourcePrivileges leftResPrivs = leftResPrivsEntry.getValue();
+            final ResourcePrivileges rightResPrivs = right.getResourceToResourcePrivileges().get(leftResPrivsEntry.getKey());
+            builder.addResourcePrivilege(leftResPrivsEntry.getKey(), leftResPrivs.getPrivileges());
+            builder.addResourcePrivilege(leftResPrivsEntry.getKey(), rightResPrivs.getPrivileges());
+        }
+        return builder.build();
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/Role.java

@@ -10,9 +10,11 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
 import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
 import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilege;
 import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivilege;
 import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege;
@@ -20,13 +22,15 @@ import org.elasticsearch.xpack.core.security.authz.privilege.Privilege;
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.function.Predicate;
 
-public final class Role {
+public class Role {
 
     public static final Role EMPTY = Role.builder("__empty").build();
 
@@ -44,6 +48,7 @@ public final class Role {
         this.runAs = Objects.requireNonNull(runAs);
     }
 
+
     public String[] names() {
         return names;
     }
@@ -72,6 +77,79 @@ public final class Role {
         return new Builder(rd, fieldPermissionsCache);
     }
 
+    /**
+     * @return A predicate that will match all the indices that this role
+     * has the privilege for executing the given action on.
+     */
+    public Predicate<String> allowedIndicesMatcher(String action) {
+        return indices().allowedIndicesMatcher(action);
+    }
+
+    /**
+     * Check if indices permissions allow for the given action
+     *
+     * @param action indices action
+     * @return {@code true} if action is allowed else returns {@code false}
+     */
+    public boolean checkIndicesAction(String action) {
+        return indices().check(action);
+    }
+
+
+    /**
+     * For given index patterns and index privileges determines allowed privileges and creates an instance of {@link ResourcePrivilegesMap}
+     * holding a map of resource to {@link ResourcePrivileges} where resource is index pattern and the map of index privilege to whether it
+     * is allowed or not.
+     *
+     * @param checkForIndexPatterns check permission grants for the set of index patterns
+     * @param allowRestrictedIndices if {@code true} then checks permission grants even for restricted indices by index matching
+     * @param checkForPrivileges check permission grants for the set of index privileges
+     * @return an instance of {@link ResourcePrivilegesMap}
+     */
+    public ResourcePrivilegesMap checkIndicesPrivileges(Set<String> checkForIndexPatterns, boolean allowRestrictedIndices,
+                                                                 Set<String> checkForPrivileges) {
+        return indices().checkResourcePrivileges(checkForIndexPatterns, allowRestrictedIndices, checkForPrivileges);
+    }
+
+    /**
+     * Check if cluster permissions allow for the given action
+     *
+     * @param action cluster action
+     * @param request {@link TransportRequest}
+     * @return {@code true} if action is allowed else returns {@code false}
+     */
+    public boolean checkClusterAction(String action, TransportRequest request) {
+        return cluster().check(action, request);
+    }
+
+    /**
+     * Check if cluster permissions grants the given cluster privilege
+     *
+     * @param clusterPrivilege cluster privilege
+     * @return {@code true} if cluster privilege is allowed else returns {@code false}
+     */
+    public boolean grants(ClusterPrivilege clusterPrivilege) {
+        return cluster().grants(clusterPrivilege);
+    }
+
+    /**
+     * For a given application, checks for the privileges for resources and returns an instance of {@link ResourcePrivilegesMap} holding a
+     * map of resource to {@link ResourcePrivileges} where the resource is application resource and the map of application privilege to
+     * whether it is allowed or not.
+     *
+     * @param applicationName checks privileges for the provided application name
+     * @param checkForResources check permission grants for the set of resources
+     * @param checkForPrivilegeNames check permission grants for the set of privilege names
+     * @param storedPrivileges stored {@link ApplicationPrivilegeDescriptor} for an application against which the access checks are
+     * performed
+     * @return an instance of {@link ResourcePrivilegesMap}
+     */
+    public ResourcePrivilegesMap checkApplicationResourcePrivileges(final String applicationName, Set<String> checkForResources,
+                                                                    Set<String> checkForPrivilegeNames,
+                                                                    Collection<ApplicationPrivilegeDescriptor> storedPrivileges) {
+        return application().checkResourcePrivileges(applicationName, checkForResources, checkForPrivilegeNames, storedPrivileges);
+    }
+
     /**
      * Returns whether at least one group encapsulated by this indices permissions is authorized to execute the
      * specified action with the requested indices/aliases. At the same time if field and/or document level security
@@ -204,4 +282,5 @@ public final class Role {
             ), Sets.newHashSet(arp.getResources()));
         }
     }
+
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/support/SecurityQueryTemplateEvaluator.java

@@ -0,0 +1,92 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.authz.support;
+
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
+import org.elasticsearch.script.TemplateScript;
+import org.elasticsearch.xpack.core.security.user.User;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Helper class that helps to evaluate the query source template.
+ */
+public final class SecurityQueryTemplateEvaluator {
+
+    private SecurityQueryTemplateEvaluator() {
+    }
+
+    /**
+     * If the query source is a template, then parses the script, compiles the
+     * script with user details parameters and then executes it to return the
+     * query string.
+     * <p>
+     * Note: This method always enforces "mustache" script language for the
+     * template.
+     *
+     * @param querySource query string template to be evaluated.
+     * @param scriptService {@link ScriptService}
+     * @param user {@link User} details for user defined parameters in the
+     * script.
+     * @return resultant query string after compiling and executing the script.
+     * If the source does not contain template then it will return the query
+     * source without any modifications.
+     * @throws IOException thrown when there is any error parsing the query
+     * string.
+     */
+    public static String evaluateTemplate(final String querySource, final ScriptService scriptService, final User user) throws IOException {
+        // EMPTY is safe here because we never use namedObject
+        try (XContentParser parser = XContentFactory.xContent(querySource).createParser(NamedXContentRegistry.EMPTY,
+                LoggingDeprecationHandler.INSTANCE, querySource)) {
+            XContentParser.Token token = parser.nextToken();
+            if (token != XContentParser.Token.START_OBJECT) {
+                throw new ElasticsearchParseException("Unexpected token [" + token + "]");
+            }
+            token = parser.nextToken();
+            if (token != XContentParser.Token.FIELD_NAME) {
+                throw new ElasticsearchParseException("Unexpected token [" + token + "]");
+            }
+            if ("template".equals(parser.currentName())) {
+                token = parser.nextToken();
+                if (token != XContentParser.Token.START_OBJECT) {
+                    throw new ElasticsearchParseException("Unexpected token [" + token + "]");
+                }
+                Script script = Script.parse(parser);
+                // Add the user details to the params
+                Map<String, Object> params = new HashMap<>();
+                if (script.getParams() != null) {
+                    params.putAll(script.getParams());
+                }
+                Map<String, Object> userModel = new HashMap<>();
+                userModel.put("username", user.principal());
+                userModel.put("full_name", user.fullName());
+                userModel.put("email", user.email());
+                userModel.put("roles", Arrays.asList(user.roles()));
+                userModel.put("metadata", Collections.unmodifiableMap(user.metadata()));
+                params.put("_user", userModel);
+                // Always enforce mustache script lang:
+                script = new Script(script.getType(), script.getType() == ScriptType.STORED ? null : "mustache", script.getIdOrCode(),
+                        script.getOptions(), params);
+                TemplateScript compiledTemplate = scriptService.compile(script, TemplateScript.CONTEXT).newInstance(script.getParams());
+                return compiledTemplate.execute();
+            } else {
+                return querySource;
+            }
+        }
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/client/SecurityClient.java

@@ -10,6 +10,16 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.xpack.core.security.action.CreateApiKeyAction;
+import org.elasticsearch.xpack.core.security.action.CreateApiKeyRequest;
+import org.elasticsearch.xpack.core.security.action.CreateApiKeyRequestBuilder;
+import org.elasticsearch.xpack.core.security.action.CreateApiKeyResponse;
+import org.elasticsearch.xpack.core.security.action.GetApiKeyAction;
+import org.elasticsearch.xpack.core.security.action.GetApiKeyRequest;
+import org.elasticsearch.xpack.core.security.action.GetApiKeyResponse;
+import org.elasticsearch.xpack.core.security.action.InvalidateApiKeyAction;
+import org.elasticsearch.xpack.core.security.action.InvalidateApiKeyRequest;
+import org.elasticsearch.xpack.core.security.action.InvalidateApiKeyResponse;
 import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesAction;
 import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesRequestBuilder;
 import org.elasticsearch.xpack.core.security.action.privilege.GetPrivilegesAction;
@@ -334,6 +344,27 @@ public class SecurityClient {
         client.execute(InvalidateTokenAction.INSTANCE, request, listener);
     }
 
+    /* -- Api Keys -- */
+    public CreateApiKeyRequestBuilder prepareCreateApiKey() {
+        return new CreateApiKeyRequestBuilder(client);
+    }
+
+    public CreateApiKeyRequestBuilder prepareCreateApiKey(BytesReference bytesReference, XContentType xContentType) throws IOException {
+        return new CreateApiKeyRequestBuilder(client).source(bytesReference, xContentType);
+    }
+
+    public void createApiKey(CreateApiKeyRequest request, ActionListener<CreateApiKeyResponse> listener) {
+        client.execute(CreateApiKeyAction.INSTANCE, request, listener);
+    }
+
+    public void invalidateApiKey(InvalidateApiKeyRequest request, ActionListener<InvalidateApiKeyResponse> listener) {
+        client.execute(InvalidateApiKeyAction.INSTANCE, request, listener);
+    }
+
+    public void getApiKey(GetApiKeyRequest request, ActionListener<GetApiKeyResponse> listener) {
+        client.execute(GetApiKeyAction.INSTANCE, request, listener);
+    }
+
     public SamlAuthenticateRequestBuilder prepareSamlAuthenticate(byte[] xmlContent, List<String> validIds) {
         final SamlAuthenticateRequestBuilder builder = new SamlAuthenticateRequestBuilder(client);
         builder.saml(xmlContent);

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Automatons.java

@@ -26,6 +26,7 @@ import java.util.function.Predicate;
 import static org.apache.lucene.util.automaton.MinimizationOperations.minimize;
 import static org.apache.lucene.util.automaton.Operations.DEFAULT_MAX_DETERMINIZED_STATES;
 import static org.apache.lucene.util.automaton.Operations.concatenate;
+import static org.apache.lucene.util.automaton.Operations.intersection;
 import static org.apache.lucene.util.automaton.Operations.minus;
 import static org.apache.lucene.util.automaton.Operations.union;
 import static org.elasticsearch.common.Strings.collectionToDelimitedString;
@@ -173,6 +174,11 @@ public final class Automatons {
         return minimize(res, maxDeterminizedStates);
     }
 
+    public static Automaton intersectAndMinimize(Automaton a1, Automaton a2) {
+        Automaton res = intersection(a1, a2);
+        return minimize(res, maxDeterminizedStates);
+    }
+
     public static Predicate<String> predicate(String... patterns) {
         return predicate(Arrays.asList(patterns));
     }

x-pack/plugin/core/src/main/resources/security-index-template.json

@@ -152,6 +152,40 @@
           "type" : "date",
           "format" : "epoch_millis"
         },
+        "api_key_hash" : {
+          "type" : "keyword",
+          "index": false,
+          "doc_values": false
+        },
+        "api_key_invalidated" : {
+          "type" : "boolean"
+        },
+        "role_descriptors" : {
+          "type" : "object",
+          "enabled": false
+        },
+        "limited_by_role_descriptors" : {
+          "type" : "object",
+          "enabled": false
+        },
+        "version" : {
+          "type" : "integer"
+        },
+        "creator" : {
+          "type" : "object",
+          "properties" : {
+            "principal" : {
+              "type": "keyword"
+            },
+            "metadata" : {
+              "type" : "object",
+              "dynamic" : true
+            },
+            "realm" : {
+              "type" : "keyword"
+            }
+          }
+        },
         "rules" : {
           "type" : "object",
           "dynamic" : true

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestBuilderTests.java

@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.action;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.IndicesPrivileges;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.mockito.Mockito.mock;
+
+public class CreateApiKeyRequestBuilderTests extends ESTestCase {
+
+    public void testParserAndCreateApiRequestBuilder() throws IOException {
+        boolean withExpiration = randomBoolean();
+        final String json = "{ \"name\" : \"my-api-key\", "
+                + ((withExpiration) ? " \"expiration\": \"1d\", " : "")
+                +" \"role_descriptors\": { \"role-a\": {\"cluster\":[\"a-1\", \"a-2\"],"
+                + " \"index\": [{\"names\": [\"indx-a\"], \"privileges\": [\"read\"] }] }, "
+                + " \"role-b\": {\"cluster\":[\"b\"],"
+                + " \"index\": [{\"names\": [\"indx-b\"], \"privileges\": [\"read\"] }] } "
+                + "} }";
+        final BytesArray source = new BytesArray(json);
+        final NodeClient mockClient = mock(NodeClient.class);
+        final CreateApiKeyRequest request = new CreateApiKeyRequestBuilder(mockClient).source(source, XContentType.JSON).request();
+        final List<RoleDescriptor> actualRoleDescriptors = request.getRoleDescriptors();
+        assertThat(request.getName(), equalTo("my-api-key"));
+        assertThat(actualRoleDescriptors.size(), is(2));
+        for (RoleDescriptor rd : actualRoleDescriptors) {
+            String[] clusters = null;
+            IndicesPrivileges indicesPrivileges = null;
+            if (rd.getName().equals("role-a")) {
+                clusters = new String[] { "a-1", "a-2" };
+                indicesPrivileges = RoleDescriptor.IndicesPrivileges.builder().indices("indx-a").privileges("read").build();
+            } else if (rd.getName().equals("role-b")){
+                clusters = new String[] { "b" };
+                indicesPrivileges = RoleDescriptor.IndicesPrivileges.builder().indices("indx-b").privileges("read").build();
+            } else {
+                fail("unexpected role name");
+            }
+            assertThat(rd.getClusterPrivileges(), arrayContainingInAnyOrder(clusters));
+            assertThat(rd.getIndicesPrivileges(),
+                    arrayContainingInAnyOrder(indicesPrivileges));
+        }
+        if (withExpiration) {
+            assertThat(request.getExpiration(), equalTo(TimeValue.parseTimeValue("1d", "expiration")));
+        }
+    }
+}

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestTests.java

@@ -0,0 +1,113 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.action;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.is;
+
+public class CreateApiKeyRequestTests extends ESTestCase {
+
+    public void testNameValidation() {
+        final String name = randomAlphaOfLengthBetween(1, 256);
+        CreateApiKeyRequest request = new CreateApiKeyRequest();
+
+        ActionRequestValidationException ve = request.validate();
+        assertNotNull(ve);
+        assertThat(ve.validationErrors().size(), is(1));
+        assertThat(ve.validationErrors().get(0), containsString("name is required"));
+
+        request.setName(name);
+        ve = request.validate();
+        assertNull(ve);
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> request.setName(""));
+        assertThat(e.getMessage(), containsString("name must not be null or empty"));
+
+        e = expectThrows(IllegalArgumentException.class, () -> request.setName(null));
+        assertThat(e.getMessage(), containsString("name must not be null or empty"));
+
+        request.setName(randomAlphaOfLength(257));
+        ve = request.validate();
+        assertNotNull(ve);
+        assertThat(ve.validationErrors().size(), is(1));
+        assertThat(ve.validationErrors().get(0), containsString("name may not be more than 256 characters long"));
+
+        request.setName(" leading space");
+        ve = request.validate();
+        assertNotNull(ve);
+        assertThat(ve.validationErrors().size(), is(1));
+        assertThat(ve.validationErrors().get(0), containsString("name may not begin or end with whitespace"));
+
+        request.setName("trailing space ");
+        ve = request.validate();
+        assertNotNull(ve);
+        assertThat(ve.validationErrors().size(), is(1));
+        assertThat(ve.validationErrors().get(0), containsString("name may not begin or end with whitespace"));
+
+        request.setName(" leading and trailing space ");
+        ve = request.validate();
+        assertNotNull(ve);
+        assertThat(ve.validationErrors().size(), is(1));
+        assertThat(ve.validationErrors().get(0), containsString("name may not begin or end with whitespace"));
+
+        request.setName("inner space");
+        ve = request.validate();
+        assertNull(ve);
+
+        request.setName("_foo");
+        ve = request.validate();
+        assertNotNull(ve);
+        assertThat(ve.validationErrors().size(), is(1));
+        assertThat(ve.validationErrors().get(0), containsString("name may not begin with an underscore"));
+    }
+
+    public void testSerialization() throws IOException {
+        final String name = randomAlphaOfLengthBetween(1, 256);
+        final TimeValue expiration = randomBoolean() ? null :
+            TimeValue.parseTimeValue(randomTimeValue(), "test serialization of create api key");
+        final WriteRequest.RefreshPolicy refreshPolicy = randomFrom(WriteRequest.RefreshPolicy.values());
+        final int numDescriptors = randomIntBetween(0, 4);
+        final List<RoleDescriptor> descriptorList = new ArrayList<>();
+        for (int i = 0; i < numDescriptors; i++) {
+            descriptorList.add(new RoleDescriptor("role_" + i, new String[] { "all" }, null, null));
+        }
+
+        final CreateApiKeyRequest request = new CreateApiKeyRequest();
+        request.setName(name);
+        request.setExpiration(expiration);
+
+        if (refreshPolicy != request.getRefreshPolicy() || randomBoolean()) {
+            request.setRefreshPolicy(refreshPolicy);
+        }
+        if (descriptorList.isEmpty() == false || randomBoolean()) {
+            request.setRoleDescriptors(descriptorList);
+        }
+
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            request.writeTo(out);
+            try (StreamInput in = out.bytes().streamInput()) {
+                final CreateApiKeyRequest serialized = new CreateApiKeyRequest(in);
+                assertEquals(name, serialized.getName());
+                assertEquals(expiration, serialized.getExpiration());
+                assertEquals(refreshPolicy, serialized.getRefreshPolicy());
+                assertEquals(descriptorList, serialized.getRoleDescriptors());
+            }
+        }
+    }
+}

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyResponseTests.java

@@ -0,0 +1,81 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.action;
+
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class CreateApiKeyResponseTests extends AbstractXContentTestCase<CreateApiKeyResponse> {
+
+    @Override
+    protected CreateApiKeyResponse doParseInstance(XContentParser parser) throws IOException {
+        return CreateApiKeyResponse.fromXContent(parser);
+    }
+
+    @Override
+    protected CreateApiKeyResponse createTestInstance() {
+        final String name = randomAlphaOfLengthBetween(1, 256);
+        final SecureString key = new SecureString(UUIDs.randomBase64UUID().toCharArray());
+        final Instant expiration = randomBoolean() ? Instant.now().plus(7L, ChronoUnit.DAYS) : null;
+        final String id = randomAlphaOfLength(100);
+        return new CreateApiKeyResponse(name, id, key, expiration);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+
+    public void testSerialization() throws IOException {
+        final CreateApiKeyResponse response = createTestInstance();
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            response.writeTo(out);
+             try (StreamInput in = out.bytes().streamInput()) {
+                CreateApiKeyResponse serialized = new CreateApiKeyResponse(in);
+                assertThat(serialized, equalTo(response));
+            }
+        }
+    }
+
+    public void testEqualsHashCode() {
+        CreateApiKeyResponse createApiKeyResponse = createTestInstance();
+
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(createApiKeyResponse, (original) -> {
+            return new CreateApiKeyResponse(original.getName(), original.getId(), original.getKey(), original.getExpiration());
+        });
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(createApiKeyResponse, (original) -> {
+            return new CreateApiKeyResponse(original.getName(), original.getId(), original.getKey(), original.getExpiration());
+        }, CreateApiKeyResponseTests::mutateTestItem);
+    }
+
+    private static CreateApiKeyResponse mutateTestItem(CreateApiKeyResponse original) {
+        switch (randomIntBetween(0, 3)) {
+        case 0:
+            return new CreateApiKeyResponse(randomAlphaOfLength(5), original.getId(), original.getKey(), original.getExpiration());
+        case 1:
+            return new CreateApiKeyResponse(original.getName(), randomAlphaOfLength(5), original.getKey(), original.getExpiration());
+        case 2:
+            return new CreateApiKeyResponse(original.getName(), original.getId(), new SecureString(UUIDs.randomBase64UUID().toCharArray()),
+                    original.getExpiration());
+        case 3:
+            return new CreateApiKeyResponse(original.getName(), original.getId(), original.getKey(), Instant.now());
+        default:
+            return new CreateApiKeyResponse(randomAlphaOfLength(5), original.getId(), original.getKey(), original.getExpiration());
+        }
+    }
+}

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/GetApiKeyRequestTests.java

@@ -0,0 +1,103 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.action;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.io.stream.InputStreamStreamInput;
+import org.elasticsearch.common.io.stream.OutputStreamStreamOutput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+
+public class GetApiKeyRequestTests extends ESTestCase {
+
+    public void testRequestValidation() {
+        GetApiKeyRequest request = GetApiKeyRequest.usingApiKeyId(randomAlphaOfLength(5));
+        ActionRequestValidationException ve = request.validate();
+        assertNull(ve);
+        request = GetApiKeyRequest.usingApiKeyName(randomAlphaOfLength(5));
+        ve = request.validate();
+        assertNull(ve);
+        request = GetApiKeyRequest.usingRealmName(randomAlphaOfLength(5));
+        ve = request.validate();
+        assertNull(ve);
+        request = GetApiKeyRequest.usingUserName(randomAlphaOfLength(5));
+        ve = request.validate();
+        assertNull(ve);
+        request = GetApiKeyRequest.usingRealmAndUserName(randomAlphaOfLength(5), randomAlphaOfLength(7));
+        ve = request.validate();
+        assertNull(ve);
+    }
+
+    public void testRequestValidationFailureScenarios() throws IOException {
+        class Dummy extends ActionRequest {
+            String realm;
+            String user;
+            String apiKeyId;
+            String apiKeyName;
+
+            Dummy(String[] a) {
+                realm = a[0];
+                user = a[1];
+                apiKeyId = a[2];
+                apiKeyName = a[3];
+            }
+
+            @Override
+            public ActionRequestValidationException validate() {
+                return null;
+            }
+
+            @Override
+            public void writeTo(StreamOutput out) throws IOException {
+                super.writeTo(out);
+                out.writeOptionalString(realm);
+                out.writeOptionalString(user);
+                out.writeOptionalString(apiKeyId);
+                out.writeOptionalString(apiKeyName);
+            }
+        }
+
+        String[][] inputs = new String[][] {
+                { randomFrom(new String[] { null, "" }), randomFrom(new String[] { null, "" }), randomFrom(new String[] { null, "" }),
+                        randomFrom(new String[] { null, "" }) },
+                { randomFrom(new String[] { null, "" }), "user", "api-kid", "api-kname" },
+                { "realm", randomFrom(new String[] { null, "" }), "api-kid", "api-kname" },
+                { "realm", "user", "api-kid", randomFrom(new String[] { null, "" }) },
+                { randomFrom(new String[] { null, "" }), randomFrom(new String[] { null, "" }), "api-kid", "api-kname" } };
+        String[][] expectedErrorMessages = new String[][] { { "One of [api key id, api key name, username, realm name] must be specified" },
+                { "username or realm name must not be specified when the api key id or api key name is specified",
+                        "only one of [api key id, api key name] can be specified" },
+                { "username or realm name must not be specified when the api key id or api key name is specified",
+                        "only one of [api key id, api key name] can be specified" },
+                { "username or realm name must not be specified when the api key id or api key name is specified" },
+                { "only one of [api key id, api key name] can be specified" } };
+
+        for (int caseNo = 0; caseNo < inputs.length; caseNo++) {
+            try (ByteArrayOutputStream bos = new ByteArrayOutputStream();
+                    OutputStreamStreamOutput osso = new OutputStreamStreamOutput(bos)) {
+                Dummy d = new Dummy(inputs[caseNo]);
+                d.writeTo(osso);
+
+                ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
+                InputStreamStreamInput issi = new InputStreamStreamInput(bis);
+
+                GetApiKeyRequest request = new GetApiKeyRequest(issi);
+                ActionRequestValidationException ve = request.validate();
+                assertNotNull(ve);
+                assertEquals(expectedErrorMessages[caseNo].length, ve.validationErrors().size());
+                assertThat(ve.validationErrors(), containsInAnyOrder(expectedErrorMessages[caseNo]));
+            }
+        }
+    }
+}

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/GetApiKeyResponseTests.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.action;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.Arrays;
+import java.util.Collections;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class GetApiKeyResponseTests extends ESTestCase {
+
+    public void testSerialization() throws IOException {
+        boolean withExpiration = randomBoolean();
+        ApiKey apiKeyInfo = createApiKeyInfo(randomAlphaOfLength(4), randomAlphaOfLength(5), Instant.now(),
+                (withExpiration) ? Instant.now() : null, false, randomAlphaOfLength(4), randomAlphaOfLength(5));
+        GetApiKeyResponse response = new GetApiKeyResponse(Collections.singletonList(apiKeyInfo));
+        try (BytesStreamOutput output = new BytesStreamOutput()) {
+            response.writeTo(output);
+            try (StreamInput input = output.bytes().streamInput()) {
+                GetApiKeyResponse serialized = new GetApiKeyResponse(input);
+                assertThat(serialized.getApiKeyInfos(), equalTo(response.getApiKeyInfos()));
+            }
+        }
+    }
+
+    public void testToXContent() throws IOException {
+        ApiKey apiKeyInfo1 = createApiKeyInfo("name1", "id-1", Instant.ofEpochMilli(100000L), Instant.ofEpochMilli(10000000L), false,
+                "user-a", "realm-x");
+        ApiKey apiKeyInfo2 = createApiKeyInfo("name2", "id-2", Instant.ofEpochMilli(100000L), Instant.ofEpochMilli(10000000L), true,
+                "user-b", "realm-y");
+        GetApiKeyResponse response = new GetApiKeyResponse(Arrays.asList(apiKeyInfo1, apiKeyInfo2));
+        XContentBuilder builder = XContentFactory.jsonBuilder();
+        response.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        assertThat(Strings.toString(builder), equalTo(
+                "{"
+                + "\"api_keys\":["
+                + "{\"id\":\"id-1\",\"name\":\"name1\",\"creation\":100000,\"expiration\":10000000,\"invalidated\":false,"
+                + "\"username\":\"user-a\",\"realm\":\"realm-x\"},"
+                + "{\"id\":\"id-2\",\"name\":\"name2\",\"creation\":100000,\"expiration\":10000000,\"invalidated\":true,"
+                + "\"username\":\"user-b\",\"realm\":\"realm-y\"}"
+                + "]"
+                + "}"));
+    }
+
+    private ApiKey createApiKeyInfo(String name, String id, Instant creation, Instant expiration, boolean invalidated, String username,
+                                        String realm) {
+        return new ApiKey(name, id, creation, expiration, invalidated, username, realm);
+    }
+}
+

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyRequestTests.java

@@ -0,0 +1,104 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.action;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.io.stream.InputStreamStreamInput;
+import org.elasticsearch.common.io.stream.OutputStreamStreamOutput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+
+public class InvalidateApiKeyRequestTests extends ESTestCase {
+
+    public void testRequestValidation() {
+        InvalidateApiKeyRequest request = InvalidateApiKeyRequest.usingApiKeyId(randomAlphaOfLength(5));
+        ActionRequestValidationException ve = request.validate();
+        assertNull(ve);
+        request = InvalidateApiKeyRequest.usingApiKeyName(randomAlphaOfLength(5));
+        ve = request.validate();
+        assertNull(ve);
+        request = InvalidateApiKeyRequest.usingRealmName(randomAlphaOfLength(5));
+        ve = request.validate();
+        assertNull(ve);
+        request = InvalidateApiKeyRequest.usingUserName(randomAlphaOfLength(5));
+        ve = request.validate();
+        assertNull(ve);
+        request = InvalidateApiKeyRequest.usingRealmAndUserName(randomAlphaOfLength(5), randomAlphaOfLength(7));
+        ve = request.validate();
+        assertNull(ve);
+    }
+
+    public void testRequestValidationFailureScenarios() throws IOException {
+        class Dummy extends ActionRequest {
+            String realm;
+            String user;
+            String apiKeyId;
+            String apiKeyName;
+
+            Dummy(String[] a) {
+                realm = a[0];
+                user = a[1];
+                apiKeyId = a[2];
+                apiKeyName = a[3];
+            }
+
+            @Override
+            public ActionRequestValidationException validate() {
+                return null;
+            }
+
+            @Override
+            public void writeTo(StreamOutput out) throws IOException {
+                super.writeTo(out);
+                out.writeOptionalString(realm);
+                out.writeOptionalString(user);
+                out.writeOptionalString(apiKeyId);
+                out.writeOptionalString(apiKeyName);
+            }
+        }
+
+        String[][] inputs = new String[][] {
+                { randomFrom(new String[] { null, "" }), randomFrom(new String[] { null, "" }), randomFrom(new String[] { null, "" }),
+                        randomFrom(new String[] { null, "" }) },
+                { randomFrom(new String[] { null, "" }), "user", "api-kid", "api-kname" },
+                { "realm", randomFrom(new String[] { null, "" }), "api-kid", "api-kname" },
+                { "realm", "user", "api-kid", randomFrom(new String[] { null, "" }) },
+                { randomFrom(new String[] { null, "" }), randomFrom(new String[] { null, "" }), "api-kid", "api-kname" } };
+        String[][] expectedErrorMessages = new String[][] { { "One of [api key id, api key name, username, realm name] must be specified" },
+                { "username or realm name must not be specified when the api key id or api key name is specified",
+                        "only one of [api key id, api key name] can be specified" },
+                { "username or realm name must not be specified when the api key id or api key name is specified",
+                        "only one of [api key id, api key name] can be specified" },
+                { "username or realm name must not be specified when the api key id or api key name is specified" },
+                { "only one of [api key id, api key name] can be specified" } };
+
+
+        for (int caseNo = 0; caseNo < inputs.length; caseNo++) {
+            try (ByteArrayOutputStream bos = new ByteArrayOutputStream();
+                    OutputStreamStreamOutput osso = new OutputStreamStreamOutput(bos)) {
+                Dummy d = new Dummy(inputs[caseNo]);
+                d.writeTo(osso);
+
+                ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
+                InputStreamStreamInput issi = new InputStreamStreamInput(bis);
+
+                InvalidateApiKeyRequest request = new InvalidateApiKeyRequest(issi);
+                ActionRequestValidationException ve = request.validate();
+                assertNotNull(ve);
+                assertEquals(expectedErrorMessages[caseNo].length, ve.validationErrors().size());
+                assertThat(ve.validationErrors(), containsInAnyOrder(expectedErrorMessages[caseNo]));
+            }
+        }
+    }
+}

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyResponseTests.java

@@ -0,0 +1,88 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.action;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+
+public class InvalidateApiKeyResponseTests extends ESTestCase {
+
+    public void testSerialization() throws IOException {
+        InvalidateApiKeyResponse response = new InvalidateApiKeyResponse(Arrays.asList("api-key-id-1"),
+                Arrays.asList("api-key-id-2", "api-key-id-3"),
+                Arrays.asList(new ElasticsearchException("error1"),
+                        new ElasticsearchException("error2")));
+        try (BytesStreamOutput output = new BytesStreamOutput()) {
+            response.writeTo(output);
+            try (StreamInput input = output.bytes().streamInput()) {
+                InvalidateApiKeyResponse serialized = new InvalidateApiKeyResponse(input);
+                assertThat(serialized.getInvalidatedApiKeys(), equalTo(response.getInvalidatedApiKeys()));
+                assertThat(serialized.getPreviouslyInvalidatedApiKeys(),
+                    equalTo(response.getPreviouslyInvalidatedApiKeys()));
+                assertThat(serialized.getErrors().size(), equalTo(response.getErrors().size()));
+                assertThat(serialized.getErrors().get(0).toString(), containsString("error1"));
+                assertThat(serialized.getErrors().get(1).toString(), containsString("error2"));
+            }
+        }
+
+        response = new InvalidateApiKeyResponse(Arrays.asList(generateRandomStringArray(20, 15, false)),
+            Arrays.asList(generateRandomStringArray(20, 15, false)),
+            Collections.emptyList());
+        try (BytesStreamOutput output = new BytesStreamOutput()) {
+            response.writeTo(output);
+            try (StreamInput input = output.bytes().streamInput()) {
+                InvalidateApiKeyResponse serialized = new InvalidateApiKeyResponse(input);
+                assertThat(serialized.getInvalidatedApiKeys(), equalTo(response.getInvalidatedApiKeys()));
+                assertThat(serialized.getPreviouslyInvalidatedApiKeys(),
+                    equalTo(response.getPreviouslyInvalidatedApiKeys()));
+                assertThat(serialized.getErrors().size(), equalTo(response.getErrors().size()));
+            }
+        }
+    }
+
+    public void testToXContent() throws IOException {
+        InvalidateApiKeyResponse response = new InvalidateApiKeyResponse(Arrays.asList("api-key-id-1"),
+                Arrays.asList("api-key-id-2", "api-key-id-3"),
+                Arrays.asList(new ElasticsearchException("error1", new IllegalArgumentException("msg - 1")),
+                        new ElasticsearchException("error2", new IllegalArgumentException("msg - 2"))));
+        XContentBuilder builder = XContentFactory.jsonBuilder();
+        response.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        assertThat(Strings.toString(builder),
+            equalTo("{" +
+                "\"invalidated_api_keys\":[\"api-key-id-1\"]," +
+                "\"previously_invalidated_api_keys\":[\"api-key-id-2\",\"api-key-id-3\"]," +
+                "\"error_count\":2," +
+                "\"error_details\":[" +
+                "{\"type\":\"exception\"," +
+                "\"reason\":\"error1\"," +
+                "\"caused_by\":{" +
+                "\"type\":\"illegal_argument_exception\"," +
+                "\"reason\":\"msg - 1\"}" +
+                "}," +
+                "{\"type\":\"exception\"," +
+                "\"reason\":\"error2\"," +
+                "\"caused_by\":" +
+                "{\"type\":\"illegal_argument_exception\"," +
+                "\"reason\":\"msg - 2\"}" +
+                "}" +
+                "]" +
+                "}"));
+    }
+}

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesResponseTests.java

@@ -18,6 +18,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.protocol.AbstractHlrcStreamableXContentTestCase;
 import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.xpack.core.security.authz.permission.ResourcePrivileges;
 import org.hamcrest.Matchers;
 
 import java.io.IOException;
@@ -59,16 +60,17 @@ public class HasPrivilegesResponseTests
     }
 
     public void testToXContent() throws Exception {
-        final HasPrivilegesResponse response = new HasPrivilegesResponse("daredevil", false,
-            Collections.singletonMap("manage", true),
-            Arrays.asList(
-                new HasPrivilegesResponse.ResourcePrivileges("staff",
-                    MapBuilder.<String, Boolean>newMapBuilder(new LinkedHashMap<>())
-                        .put("read", true).put("index", true).put("delete", false).put("manage", false).map()),
-                new HasPrivilegesResponse.ResourcePrivileges("customers",
-                    MapBuilder.<String, Boolean>newMapBuilder(new LinkedHashMap<>())
-                        .put("read", true).put("index", true).put("delete", true).put("manage", false).map())
-            ), Collections.emptyMap());
+        final HasPrivilegesResponse response = new HasPrivilegesResponse("daredevil", false, Collections.singletonMap("manage", true),
+                Arrays.asList(
+                        ResourcePrivileges.builder("staff")
+                                .addPrivileges(MapBuilder.<String, Boolean>newMapBuilder(new LinkedHashMap<>()).put("read", true)
+                                        .put("index", true).put("delete", false).put("manage", false).map())
+                                .build(),
+                        ResourcePrivileges.builder("customers")
+                                .addPrivileges(MapBuilder.<String, Boolean>newMapBuilder(new LinkedHashMap<>()).put("read", true)
+                                        .put("index", true).put("delete", true).put("manage", false).map())
+                                .build()),
+                Collections.emptyMap());
 
         final XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent());
         response.toXContent(builder, ToXContent.EMPTY_PARAMS);
@@ -120,9 +122,9 @@ public class HasPrivilegesResponseTests
             );
     }
 
-    private static List<HasPrivilegesResponse.ResourcePrivileges> toResourcePrivileges(Map<String, Map<String, Boolean>> map) {
+    private static List<ResourcePrivileges> toResourcePrivileges(Map<String, Map<String, Boolean>> map) {
         return map.entrySet().stream()
-            .map(e -> new HasPrivilegesResponse.ResourcePrivileges(e.getKey(), e.getValue()))
+            .map(e -> ResourcePrivileges.builder(e.getKey()).addPrivileges(e.getValue()).build())
             .collect(Collectors.toList());
     }
 
@@ -146,23 +148,23 @@ public class HasPrivilegesResponseTests
         for (String priv : randomArray(1, 6, String[]::new, () -> randomAlphaOfLengthBetween(3, 12))) {
             cluster.put(priv, randomBoolean());
         }
-        final Collection<HasPrivilegesResponse.ResourcePrivileges> index = randomResourcePrivileges();
-        final Map<String, Collection<HasPrivilegesResponse.ResourcePrivileges>> application = new HashMap<>();
+        final Collection<ResourcePrivileges> index = randomResourcePrivileges();
+        final Map<String, Collection<ResourcePrivileges>> application = new HashMap<>();
         for (String app : randomArray(1, 3, String[]::new, () -> randomAlphaOfLengthBetween(3, 6).toLowerCase(Locale.ROOT))) {
             application.put(app, randomResourcePrivileges());
         }
         return new HasPrivilegesResponse(username, randomBoolean(), cluster, index, application);
     }
 
-    private Collection<HasPrivilegesResponse.ResourcePrivileges> randomResourcePrivileges() {
-        final Collection<HasPrivilegesResponse.ResourcePrivileges> list = new ArrayList<>();
+    private Collection<ResourcePrivileges> randomResourcePrivileges() {
+        final Collection<ResourcePrivileges> list = new ArrayList<>();
         // Use hash set to force a unique set of resources
         for (String resource : Sets.newHashSet(randomArray(1, 3, String[]::new, () -> randomAlphaOfLengthBetween(2, 6)))) {
             final Map<String, Boolean> privileges = new HashMap<>();
             for (String priv : randomArray(1, 5, String[]::new, () -> randomAlphaOfLengthBetween(3, 8))) {
                 privileges.put(priv, randomBoolean());
             }
-            list.add(new HasPrivilegesResponse.ResourcePrivileges(resource, privileges));
+            list.add(ResourcePrivileges.builder(resource).addPrivileges(privileges).build());
         }
         return list;
     }

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandlerTests.java

@@ -122,8 +122,9 @@ public class DefaultAuthenticationFailureHandlerTests extends ESTestCase {
         final String basicAuthScheme = "Basic realm=\"" + XPackField.SECURITY + "\" charset=\"UTF-8\"";
         final String bearerAuthScheme = "Bearer realm=\"" + XPackField.SECURITY + "\"";
         final String negotiateAuthScheme = randomFrom("Negotiate", "Negotiate Ijoijksdk");
+        final String apiKeyAuthScheme = "ApiKey";
         final Map<String, List<String>> failureResponeHeaders = new HashMap<>();
-        final List<String> supportedSchemes = Arrays.asList(basicAuthScheme, bearerAuthScheme, negotiateAuthScheme);
+        final List<String> supportedSchemes = Arrays.asList(basicAuthScheme, bearerAuthScheme, negotiateAuthScheme, apiKeyAuthScheme);
         Collections.shuffle(supportedSchemes, random());
         failureResponeHeaders.put("WWW-Authenticate", supportedSchemes);
         final DefaultAuthenticationFailureHandler failuerHandler = new DefaultAuthenticationFailureHandler(failureResponeHeaders);
@@ -134,7 +135,7 @@ public class DefaultAuthenticationFailureHandlerTests extends ESTestCase {
         assertThat(ese, is(notNullValue()));
         assertThat(ese.getHeader("WWW-Authenticate"), is(notNullValue()));
         assertThat(ese.getMessage(), equalTo("error attempting to authenticate request"));
-        assertWWWAuthenticateWithSchemes(ese, negotiateAuthScheme, bearerAuthScheme, basicAuthScheme);
+        assertWWWAuthenticateWithSchemes(ese, negotiateAuthScheme, bearerAuthScheme, apiKeyAuthScheme, basicAuthScheme);
     }
 
     private void assertWWWAuthenticateWithSchemes(final ElasticsearchSecurityException ese, final String... schemes) {

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapperIntegrationTests.java

@@ -8,6 +8,7 @@ package org.elasticsearch.xpack.core.security.authz.accesscontrol;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Field.Store;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexWriter;
@@ -16,12 +17,14 @@ import org.apache.lucene.index.NoMergePolicy;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TotalHitCountCollector;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.Accountable;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -36,14 +39,21 @@ import org.elasticsearch.index.query.TermsQueryBuilder;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.AbstractBuilderTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
+import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authc.AuthenticationField;
+import org.elasticsearch.xpack.core.security.authz.permission.DocumentPermissions;
 import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissions;
+import org.elasticsearch.xpack.core.security.user.User;
 
 import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
 
 import static java.util.Collections.singleton;
 import static java.util.Collections.singletonMap;
+import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.equalTo;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.anyString;
@@ -52,7 +62,7 @@ import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.when;
 
-public class SecurityIndexSearcherWrapperIntegrationTests extends ESTestCase {
+public class SecurityIndexSearcherWrapperIntegrationTests extends AbstractBuilderTestCase {
 
     public void testDLS() throws Exception {
         ShardId shardId = new ShardId("_index", "_na_", 0);
@@ -63,9 +73,12 @@ public class SecurityIndexSearcherWrapperIntegrationTests extends ESTestCase {
                 .then(invocationOnMock -> Collections.singletonList((String) invocationOnMock.getArguments()[0]));
 
         ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
+        final Authentication authentication = mock(Authentication.class);
+        when(authentication.getUser()).thenReturn(mock(User.class));
+        threadContext.putTransient(AuthenticationField.AUTHENTICATION_KEY, authentication);
         IndicesAccessControl.IndexAccessControl indexAccessControl = new IndicesAccessControl.IndexAccessControl(true, new
                 FieldPermissions(),
-                singleton(new BytesArray("{\"match_all\" : {}}")));
+                DocumentPermissions.filteredBy(singleton(new BytesArray("{\"match_all\" : {}}"))));
         IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(shardId.getIndex(), Settings.EMPTY);
         Client client = mock(Client.class);
         when(client.settings()).thenReturn(Settings.EMPTY);
@@ -158,4 +171,116 @@ public class SecurityIndexSearcherWrapperIntegrationTests extends ESTestCase {
         directoryReader.close();
         directory.close();
     }
+
+    public void testDLSWithLimitedPermissions() throws Exception {
+        ShardId shardId = new ShardId("_index", "_na_", 0);
+        MapperService mapperService = mock(MapperService.class);
+        ScriptService  scriptService = mock(ScriptService.class);
+        when(mapperService.documentMapper()).thenReturn(null);
+        when(mapperService.simpleMatchToFullName(anyString()))
+                .then(invocationOnMock -> Collections.singletonList((String) invocationOnMock.getArguments()[0]));
+
+        ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
+        final Authentication authentication = mock(Authentication.class);
+        when(authentication.getUser()).thenReturn(mock(User.class));
+        threadContext.putTransient(AuthenticationField.AUTHENTICATION_KEY, authentication);
+        final boolean noFilteredIndexPermissions = randomBoolean();
+        boolean restrictiveLimitedIndexPermissions = false;
+        if (noFilteredIndexPermissions == false) {
+            restrictiveLimitedIndexPermissions = randomBoolean();
+        }
+        Set<BytesReference> queries = new HashSet<>();
+        queries.add(new BytesArray("{\"terms\" : { \"f2\" : [\"fv22\"] } }"));
+        queries.add(new BytesArray("{\"terms\" : { \"f2\" : [\"fv32\"] } }"));
+        IndicesAccessControl.IndexAccessControl indexAccessControl = new IndicesAccessControl.IndexAccessControl(true, new
+                FieldPermissions(),
+                DocumentPermissions.filteredBy(queries));
+        queries = singleton(new BytesArray("{\"terms\" : { \"f1\" : [\"fv11\", \"fv21\", \"fv31\"] } }"));
+        if (restrictiveLimitedIndexPermissions) {
+            queries = singleton(new BytesArray("{\"terms\" : { \"f1\" : [\"fv11\", \"fv31\"] } }"));
+        }
+        IndicesAccessControl.IndexAccessControl limitedIndexAccessControl = new IndicesAccessControl.IndexAccessControl(true, new
+                FieldPermissions(),
+                DocumentPermissions.filteredBy(queries));
+        IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(shardId.getIndex(), Settings.EMPTY);
+        Client client = mock(Client.class);
+        when(client.settings()).thenReturn(Settings.EMPTY);
+        final long nowInMillis = randomNonNegativeLong();
+        QueryShardContext realQueryShardContext = new QueryShardContext(shardId.id(), indexSettings, null, null, mapperService, null,
+                null, xContentRegistry(), writableRegistry(), client, null, () -> nowInMillis, null);
+        QueryShardContext queryShardContext = spy(realQueryShardContext);
+        IndexSettings settings = IndexSettingsModule.newIndexSettings("_index", Settings.EMPTY);
+        BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(settings, new BitsetFilterCache.Listener() {
+            @Override
+            public void onCache(ShardId shardId, Accountable accountable) {
+            }
+
+            @Override
+            public void onRemoval(ShardId shardId, Accountable accountable) {
+            }
+        });
+
+        XPackLicenseState licenseState = mock(XPackLicenseState.class);
+        when(licenseState.isDocumentAndFieldLevelSecurityAllowed()).thenReturn(true);
+        SecurityIndexSearcherWrapper wrapper = new SecurityIndexSearcherWrapper(s -> queryShardContext,
+                bitsetFilterCache, threadContext, licenseState, scriptService) {
+
+            @Override
+            protected IndicesAccessControl getIndicesAccessControl() {
+                IndicesAccessControl indicesAccessControl = new IndicesAccessControl(true, singletonMap("_index", indexAccessControl));
+                if (noFilteredIndexPermissions) {
+                    return indicesAccessControl;
+                }
+                IndicesAccessControl limitedByIndicesAccessControl = new IndicesAccessControl(true,
+                        singletonMap("_index", limitedIndexAccessControl));
+                return indicesAccessControl.limitIndicesAccessControl(limitedByIndicesAccessControl);
+            }
+        };
+
+        Directory directory = newDirectory();
+        IndexWriter iw = new IndexWriter(
+                directory,
+                new IndexWriterConfig(new StandardAnalyzer()).setMergePolicy(NoMergePolicy.INSTANCE)
+        );
+
+        Document doc1 = new Document();
+        doc1.add(new StringField("f1", "fv11", Store.NO));
+        doc1.add(new StringField("f2", "fv12", Store.NO));
+        iw.addDocument(doc1);
+        Document doc2 = new Document();
+        doc2.add(new StringField("f1", "fv21", Store.NO));
+        doc2.add(new StringField("f2", "fv22", Store.NO));
+        iw.addDocument(doc2);
+        Document doc3 = new Document();
+        doc3.add(new StringField("f1", "fv31", Store.NO));
+        doc3.add(new StringField("f2", "fv32", Store.NO));
+        iw.addDocument(doc3);
+        iw.commit();
+        iw.close();
+
+        DirectoryReader directoryReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(directory), shardId);
+        DirectoryReader wrappedDirectoryReader = wrapper.wrap(directoryReader);
+        IndexSearcher indexSearcher = wrapper.wrap(new IndexSearcher(wrappedDirectoryReader));
+
+        ScoreDoc[] hits = indexSearcher.search(new MatchAllDocsQuery(), 1000).scoreDocs;
+        Set<Integer> actualDocIds = new HashSet<>();
+        for (ScoreDoc doc : hits) {
+            actualDocIds.add(doc.doc);
+        }
+
+        if (noFilteredIndexPermissions) {
+            assertThat(actualDocIds, containsInAnyOrder(1, 2));
+        } else {
+            if (restrictiveLimitedIndexPermissions) {
+                assertThat(actualDocIds, containsInAnyOrder(2));
+            } else {
+                assertThat(actualDocIds, containsInAnyOrder(1, 2));
+            }
+        }
+
+        bitsetFilterCache.close();
+        directoryReader.close();
+        directory.close();
+    }
+
 }

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapperUnitTests.java

@@ -28,21 +28,16 @@ import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.Weight;
-import org.apache.lucene.search.join.ScoreMode;
 import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.store.MMapDirectory;
 import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.BitSet;
 import org.apache.lucene.util.FixedBitSet;
-import org.elasticsearch.core.internal.io.IOUtils;
 import org.apache.lucene.util.SparseFixedBitSet;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
@@ -50,59 +45,35 @@ import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.SeqNoFieldMapper;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
-import org.elasticsearch.index.query.BoolQueryBuilder;
-import org.elasticsearch.index.query.BoostingQueryBuilder;
-import org.elasticsearch.index.query.ConstantScoreQueryBuilder;
-import org.elasticsearch.index.query.GeoShapeQueryBuilder;
-import org.elasticsearch.index.query.MatchAllQueryBuilder;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.index.query.QueryRewriteContext;
-import org.elasticsearch.index.query.TermQueryBuilder;
-import org.elasticsearch.index.query.TermsQueryBuilder;
-import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.indices.TermsLookup;
-import org.elasticsearch.join.query.HasChildQueryBuilder;
-import org.elasticsearch.join.query.HasParentQueryBuilder;
 import org.elasticsearch.license.XPackLicenseState;
-import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.script.ScriptType;
-import org.elasticsearch.script.TemplateScript;
 import org.elasticsearch.search.aggregations.LeafBucketCollector;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 import org.elasticsearch.xpack.core.security.authz.accesscontrol.DocumentSubsetReader.DocumentSubsetDirectoryReader;
+import org.elasticsearch.xpack.core.security.authz.permission.DocumentPermissions;
 import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissions;
 import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsDefinition;
-import org.elasticsearch.xpack.core.security.user.User;
 import org.junit.After;
 import org.junit.Before;
-import org.mockito.ArgumentCaptor;
 
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.IdentityHashMap;
-import java.util.Map;
 import java.util.Set;
 
 import static java.util.Collections.singletonMap;
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.xpack.core.security.authz.accesscontrol.SecurityIndexSearcherWrapper.intersectScorerAndRoleBits;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.sameInstance;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyZeroInteractions;
 import static org.mockito.Mockito.when;
 
 public class SecurityIndexSearcherWrapperUnitTests extends ESTestCase {
@@ -136,7 +107,7 @@ public class SecurityIndexSearcherWrapperUnitTests extends ESTestCase {
         IndexShard indexShard = mock(IndexShard.class);
         when(indexShard.shardId()).thenReturn(shardId);
 
-        Directory directory = new RAMDirectory();
+        Directory directory = new MMapDirectory(createTempDir());
         IndexWriter writer = new IndexWriter(directory, newIndexWriterConfig());
         writer.close();
 
@@ -156,7 +127,7 @@ public class SecurityIndexSearcherWrapperUnitTests extends ESTestCase {
             @Override
             protected IndicesAccessControl getIndicesAccessControl() {
                 IndicesAccessControl.IndexAccessControl indexAccessControl = new IndicesAccessControl.IndexAccessControl(true,
-                        new FieldPermissions(fieldPermissionDef(new String[]{}, null)), null);
+                        new FieldPermissions(fieldPermissionDef(new String[]{}, null)), DocumentPermissions.allowAll());
                 return new IndicesAccessControl(true, singletonMap("_index", indexAccessControl));
             }
         };
@@ -423,66 +394,6 @@ public class SecurityIndexSearcherWrapperUnitTests extends ESTestCase {
         doTestIndexSearcherWrapper(false, true);
     }
 
-    public void testTemplating() throws Exception {
-        User user = new User("_username", new String[]{"role1", "role2"}, "_full_name", "_email",
-                Collections.singletonMap("key", "value"), true);
-        securityIndexSearcherWrapper =
-                new SecurityIndexSearcherWrapper(null, null, threadContext, licenseState, scriptService) {
-
-                    @Override
-                    protected User getUser() {
-                        return user;
-                    }
-                };
-
-        TemplateScript.Factory compiledTemplate = templateParams ->
-                new TemplateScript(templateParams) {
-                    @Override
-                    public String execute() {
-                        return "rendered_text";
-                    }
-                };
-
-        when(scriptService.compile(any(Script.class), eq(TemplateScript.CONTEXT))).thenReturn(compiledTemplate);
-
-        XContentBuilder builder = jsonBuilder();
-        String query = Strings.toString(new TermQueryBuilder("field", "{{_user.username}}").toXContent(builder, ToXContent.EMPTY_PARAMS));
-        Script script = new Script(ScriptType.INLINE, "mustache", query, Collections.singletonMap("custom", "value"));
-        builder = jsonBuilder().startObject().field("template");
-        script.toXContent(builder, ToXContent.EMPTY_PARAMS);
-        String querySource = Strings.toString(builder.endObject());
-
-        securityIndexSearcherWrapper.evaluateTemplate(querySource);
-        ArgumentCaptor<Script> argument = ArgumentCaptor.forClass(Script.class);
-        verify(scriptService).compile(argument.capture(), eq(TemplateScript.CONTEXT));
-        Script usedScript = argument.getValue();
-        assertThat(usedScript.getIdOrCode(), equalTo(script.getIdOrCode()));
-        assertThat(usedScript.getType(), equalTo(script.getType()));
-        assertThat(usedScript.getLang(), equalTo("mustache"));
-        assertThat(usedScript.getOptions(), equalTo(script.getOptions()));
-        assertThat(usedScript.getParams().size(), equalTo(2));
-        assertThat(usedScript.getParams().get("custom"), equalTo("value"));
-
-        Map<String, Object> userModel = new HashMap<>();
-        userModel.put("username", user.principal());
-        userModel.put("full_name", user.fullName());
-        userModel.put("email", user.email());
-        userModel.put("roles", Arrays.asList(user.roles()));
-        userModel.put("metadata", user.metadata());
-        assertThat(usedScript.getParams().get("_user"), equalTo(userModel));
-
-    }
-
-    public void testSkipTemplating() throws Exception {
-        securityIndexSearcherWrapper =
-                new SecurityIndexSearcherWrapper(null, null, threadContext, licenseState, scriptService);
-        XContentBuilder builder = jsonBuilder();
-        String querySource =  Strings.toString(new TermQueryBuilder("field", "value").toXContent(builder, ToXContent.EMPTY_PARAMS));
-        String result = securityIndexSearcherWrapper.evaluateTemplate(querySource);
-        assertThat(result, sameInstance(querySource));
-        verifyZeroInteractions(scriptService);
-    }
-
     static class CreateScorerOnceWeight extends Weight {
 
         private final Weight weight;
@@ -622,59 +533,6 @@ public class SecurityIndexSearcherWrapperUnitTests extends ESTestCase {
         IOUtils.close(reader, w, dir);
     }
 
-    public void testVerifyRoleQuery() throws Exception {
-        QueryBuilder queryBuilder1 = new TermsQueryBuilder("field", "val1", "val2");
-        SecurityIndexSearcherWrapper.verifyRoleQuery(queryBuilder1);
-
-        QueryBuilder queryBuilder2 = new TermsQueryBuilder("field", new TermsLookup("_index", "_type", "_id", "_path"));
-        Exception e = expectThrows(IllegalArgumentException.class, () -> SecurityIndexSearcherWrapper.verifyRoleQuery(queryBuilder2));
-        assertThat(e.getMessage(), equalTo("terms query with terms lookup isn't supported as part of a role query"));
-
-        QueryBuilder queryBuilder3 = new GeoShapeQueryBuilder("field", "_id", "_type");
-        e = expectThrows(IllegalArgumentException.class, () -> SecurityIndexSearcherWrapper.verifyRoleQuery(queryBuilder3));
-        assertThat(e.getMessage(), equalTo("geoshape query referring to indexed shapes isn't support as part of a role query"));
-
-        QueryBuilder queryBuilder4 = new HasChildQueryBuilder("_type", new MatchAllQueryBuilder(), ScoreMode.None);
-        e = expectThrows(IllegalArgumentException.class, () -> SecurityIndexSearcherWrapper.verifyRoleQuery(queryBuilder4));
-        assertThat(e.getMessage(), equalTo("has_child query isn't support as part of a role query"));
-
-        QueryBuilder queryBuilder5 = new HasParentQueryBuilder("_type", new MatchAllQueryBuilder(), false);
-        e = expectThrows(IllegalArgumentException.class, () -> SecurityIndexSearcherWrapper.verifyRoleQuery(queryBuilder5));
-        assertThat(e.getMessage(), equalTo("has_parent query isn't support as part of a role query"));
-
-        QueryBuilder queryBuilder6 = new BoolQueryBuilder().must(new GeoShapeQueryBuilder("field", "_id", "_type"));
-        e = expectThrows(IllegalArgumentException.class, () -> SecurityIndexSearcherWrapper.verifyRoleQuery(queryBuilder6));
-        assertThat(e.getMessage(), equalTo("geoshape query referring to indexed shapes isn't support as part of a role query"));
-
-        QueryBuilder queryBuilder7 = new ConstantScoreQueryBuilder(new GeoShapeQueryBuilder("field", "_id", "_type"));
-        e = expectThrows(IllegalArgumentException.class, () -> SecurityIndexSearcherWrapper.verifyRoleQuery(queryBuilder7));
-        assertThat(e.getMessage(), equalTo("geoshape query referring to indexed shapes isn't support as part of a role query"));
-
-        QueryBuilder queryBuilder8 = new FunctionScoreQueryBuilder(new GeoShapeQueryBuilder("field", "_id", "_type"));
-        e = expectThrows(IllegalArgumentException.class, () -> SecurityIndexSearcherWrapper.verifyRoleQuery(queryBuilder8));
-        assertThat(e.getMessage(), equalTo("geoshape query referring to indexed shapes isn't support as part of a role query"));
-
-        QueryBuilder queryBuilder9 = new BoostingQueryBuilder(new GeoShapeQueryBuilder("field", "_id", "_type"),
-                new MatchAllQueryBuilder());
-        e = expectThrows(IllegalArgumentException.class, () -> SecurityIndexSearcherWrapper.verifyRoleQuery(queryBuilder9));
-        assertThat(e.getMessage(), equalTo("geoshape query referring to indexed shapes isn't support as part of a role query"));
-    }
-
-    public void testFailIfQueryUsesClient() throws Exception {
-        Client client = mock(Client.class);
-        when(client.settings()).thenReturn(Settings.EMPTY);
-        final long nowInMillis = randomNonNegativeLong();
-        QueryRewriteContext context = new QueryRewriteContext(xContentRegistry(), writableRegistry(), client,
-                () -> nowInMillis);
-        QueryBuilder queryBuilder1 = new TermsQueryBuilder("field", "val1", "val2");
-        SecurityIndexSearcherWrapper.failIfQueryUsesClient(queryBuilder1, context);
-
-        QueryBuilder queryBuilder2 = new TermsQueryBuilder("field", new TermsLookup("_index", "_type", "_id", "_path"));
-        Exception e = expectThrows(IllegalStateException.class,
-                () -> SecurityIndexSearcherWrapper.failIfQueryUsesClient(queryBuilder2, context));
-        assertThat(e.getMessage(), equalTo("role queries are not allowed to execute additional requests"));
-    }
-
     private static FieldPermissionsDefinition fieldPermissionDef(String[] granted, String[] denied) {
         return new FieldPermissionsDefinition(granted, denied);
     }

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/permission/DocumentPermissionsTests.java

@@ -0,0 +1,123 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.authz.permission;
+
+import org.apache.lucene.search.join.ScoreMode;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.BoostingQueryBuilder;
+import org.elasticsearch.index.query.ConstantScoreQueryBuilder;
+import org.elasticsearch.index.query.GeoShapeQueryBuilder;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryRewriteContext;
+import org.elasticsearch.index.query.TermsQueryBuilder;
+import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
+import org.elasticsearch.indices.TermsLookup;
+import org.elasticsearch.join.query.HasChildQueryBuilder;
+import org.elasticsearch.join.query.HasParentQueryBuilder;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class DocumentPermissionsTests extends ESTestCase {
+
+    public void testHasDocumentPermissions() throws IOException {
+        final DocumentPermissions documentPermissions1 = DocumentPermissions.allowAll();
+        assertThat(documentPermissions1, is(notNullValue()));
+        assertThat(documentPermissions1.hasDocumentLevelPermissions(), is(false));
+        assertThat(documentPermissions1.filter(null, null, null, null), is(nullValue()));
+
+        Set<BytesReference> queries = Collections.singleton(new BytesArray("{\"match_all\" : {}}"));
+        final DocumentPermissions documentPermissions2 = DocumentPermissions
+                .filteredBy(queries);
+        assertThat(documentPermissions2, is(notNullValue()));
+        assertThat(documentPermissions2.hasDocumentLevelPermissions(), is(true));
+        assertThat(documentPermissions2.getQueries(), equalTo(queries));
+
+        final DocumentPermissions documentPermissions3 = documentPermissions1.limitDocumentPermissions(documentPermissions2);
+        assertThat(documentPermissions3, is(notNullValue()));
+        assertThat(documentPermissions3.hasDocumentLevelPermissions(), is(true));
+        assertThat(documentPermissions3.getQueries(), is(nullValue()));
+        assertThat(documentPermissions3.getLimitedByQueries(), equalTo(queries));
+
+        final DocumentPermissions documentPermissions4 = DocumentPermissions.allowAll()
+                .limitDocumentPermissions(DocumentPermissions.allowAll());
+        assertThat(documentPermissions4, is(notNullValue()));
+        assertThat(documentPermissions4.hasDocumentLevelPermissions(), is(false));
+
+        AssertionError ae = expectThrows(AssertionError.class,
+                () -> DocumentPermissions.allowAll().limitDocumentPermissions(documentPermissions3));
+        assertThat(ae.getMessage(), containsString("nested scoping for document permissions is not permitted"));
+    }
+
+    public void testVerifyRoleQuery() throws Exception {
+        QueryBuilder queryBuilder1 = new TermsQueryBuilder("field", "val1", "val2");
+        DocumentPermissions.verifyRoleQuery(queryBuilder1);
+
+        QueryBuilder queryBuilder2 = new TermsQueryBuilder("field", new TermsLookup("_index", "_type", "_id", "_path"));
+        Exception e = expectThrows(IllegalArgumentException.class, () -> DocumentPermissions.verifyRoleQuery(queryBuilder2));
+        assertThat(e.getMessage(), equalTo("terms query with terms lookup isn't supported as part of a role query"));
+
+        QueryBuilder queryBuilder3 = new GeoShapeQueryBuilder("field", "_id", "_type");
+        e = expectThrows(IllegalArgumentException.class, () -> DocumentPermissions.verifyRoleQuery(queryBuilder3));
+        assertThat(e.getMessage(), equalTo("geoshape query referring to indexed shapes isn't support as part of a role query"));
+
+        QueryBuilder queryBuilder4 = new HasChildQueryBuilder("_type", new MatchAllQueryBuilder(), ScoreMode.None);
+        e = expectThrows(IllegalArgumentException.class, () -> DocumentPermissions.verifyRoleQuery(queryBuilder4));
+        assertThat(e.getMessage(), equalTo("has_child query isn't support as part of a role query"));
+
+        QueryBuilder queryBuilder5 = new HasParentQueryBuilder("_type", new MatchAllQueryBuilder(), false);
+        e = expectThrows(IllegalArgumentException.class, () -> DocumentPermissions.verifyRoleQuery(queryBuilder5));
+        assertThat(e.getMessage(), equalTo("has_parent query isn't support as part of a role query"));
+
+        QueryBuilder queryBuilder6 = new BoolQueryBuilder().must(new GeoShapeQueryBuilder("field", "_id", "_type"));
+        e = expectThrows(IllegalArgumentException.class, () -> DocumentPermissions.verifyRoleQuery(queryBuilder6));
+        assertThat(e.getMessage(), equalTo("geoshape query referring to indexed shapes isn't support as part of a role query"));
+
+        QueryBuilder queryBuilder7 = new ConstantScoreQueryBuilder(new GeoShapeQueryBuilder("field", "_id", "_type"));
+        e = expectThrows(IllegalArgumentException.class, () -> DocumentPermissions.verifyRoleQuery(queryBuilder7));
+        assertThat(e.getMessage(), equalTo("geoshape query referring to indexed shapes isn't support as part of a role query"));
+
+        QueryBuilder queryBuilder8 = new FunctionScoreQueryBuilder(new GeoShapeQueryBuilder("field", "_id", "_type"));
+        e = expectThrows(IllegalArgumentException.class, () -> DocumentPermissions.verifyRoleQuery(queryBuilder8));
+        assertThat(e.getMessage(), equalTo("geoshape query referring to indexed shapes isn't support as part of a role query"));
+
+        QueryBuilder queryBuilder9 = new BoostingQueryBuilder(new GeoShapeQueryBuilder("field", "_id", "_type"),
+                new MatchAllQueryBuilder());
+        e = expectThrows(IllegalArgumentException.class, () -> DocumentPermissions.verifyRoleQuery(queryBuilder9));
+        assertThat(e.getMessage(), equalTo("geoshape query referring to indexed shapes isn't support as part of a role query"));
+    }
+
+    public void testFailIfQueryUsesClient() throws Exception {
+        Client client = mock(Client.class);
+        when(client.settings()).thenReturn(Settings.EMPTY);
+        final long nowInMillis = randomNonNegativeLong();
+        QueryRewriteContext context = new QueryRewriteContext(xContentRegistry(), writableRegistry(), client,
+                () -> nowInMillis);
+        QueryBuilder queryBuilder1 = new TermsQueryBuilder("field", "val1", "val2");
+        DocumentPermissions.failIfQueryUsesClient(queryBuilder1, context);
+
+        QueryBuilder queryBuilder2 = new TermsQueryBuilder("field", new TermsLookup("_index", "_type", "_id", "_path"));
+        Exception e = expectThrows(IllegalStateException.class,
+                () -> DocumentPermissions.failIfQueryUsesClient(queryBuilder2, context));
+        assertThat(e.getMessage(), equalTo("role queries are not allowed to execute additional requests"));
+    }
+}

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissionsTests.java

@@ -0,0 +1,81 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.authz.permission;
+
+import org.apache.lucene.util.automaton.CharacterRunAutomaton;
+import org.elasticsearch.test.ESTestCase;
+import org.hamcrest.core.IsSame;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.mockito.Matchers.same;
+
+public class FieldPermissionsTests extends ESTestCase {
+
+    public void testFieldPermissionsIntersection() throws IOException {
+
+        final FieldPermissions fieldPermissions = FieldPermissions.DEFAULT;
+        final FieldPermissions fieldPermissions1 = new FieldPermissions(
+                fieldPermissionDef(new String[] { "f1", "f2", "f3*" }, new String[] { "f3" }));
+        final FieldPermissions fieldPermissions2 = new FieldPermissions(
+                fieldPermissionDef(new String[] { "f1", "f3*", "f4" }, new String[] { "f3" }));
+
+        {
+            FieldPermissions result = fieldPermissions.limitFieldPermissions(randomFrom(new FieldPermissions(), null));
+            assertThat(result, is(notNullValue()));
+            assertThat(result, IsSame.sameInstance(FieldPermissions.DEFAULT));
+        }
+
+        {
+            FieldPermissions result = fieldPermissions1.limitFieldPermissions(new FieldPermissions());
+            assertThat(result, is(notNullValue()));
+            assertThat(result, not(same(fieldPermissions)));
+            assertThat(result, not(same(fieldPermissions1)));
+            CharacterRunAutomaton automaton = new CharacterRunAutomaton(result.getIncludeAutomaton());
+            assertThat(automaton.run("f1"), is(true));
+            assertThat(automaton.run("f2"), is(true));
+            assertThat(automaton.run("f3"), is(false));
+            assertThat(automaton.run("f31"), is(true));
+            assertThat(automaton.run("f4"), is(false));
+        }
+
+        {
+            FieldPermissions result = fieldPermissions1.limitFieldPermissions(fieldPermissions2);
+            assertThat(result, is(notNullValue()));
+            assertThat(result, not(same(fieldPermissions1)));
+            assertThat(result, not(same(fieldPermissions2)));
+            CharacterRunAutomaton automaton = new CharacterRunAutomaton(result.getIncludeAutomaton());
+            assertThat(automaton.run("f1"), is(true));
+            assertThat(automaton.run("f2"), is(false));
+            assertThat(automaton.run("f3"), is(false));
+            assertThat(automaton.run("f31"), is(true));
+            assertThat(automaton.run("f4"), is(false));
+        }
+
+        {
+            FieldPermissions result = fieldPermissions.limitFieldPermissions(fieldPermissions2);
+            assertThat(result, is(notNullValue()));
+            assertThat(result, not(same(fieldPermissions1)));
+            assertThat(result, not(same(fieldPermissions2)));
+            CharacterRunAutomaton automaton = new CharacterRunAutomaton(result.getIncludeAutomaton());
+            assertThat(automaton.run("f1"), is(true));
+            assertThat(automaton.run("f2"), is(false));
+            assertThat(automaton.run("f3"), is(false));
+            assertThat(automaton.run("f31"), is(true));
+            assertThat(automaton.run("f4"), is(true));
+            assertThat(automaton.run("f5"), is(false));
+        }
+    }
+
+    private static FieldPermissionsDefinition fieldPermissionDef(String[] granted, String[] denied) {
+        return new FieldPermissionsDefinition(granted, denied);
+    }
+
+}

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/permission/LimitedRoleTests.java

@@ -0,0 +1,403 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.authz.permission;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
+import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction;
+import org.elasticsearch.action.search.SearchAction;
+import org.elasticsearch.client.security.user.privileges.Role.ClusterPrivilegeName;
+import org.elasticsearch.cluster.metadata.AliasMetaData;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.collect.MapBuilder;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.transport.TransportRequest;
+import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
+import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilege;
+import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege;
+import org.junit.Before;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.mockito.Mockito.mock;
+
+public class LimitedRoleTests extends ESTestCase {
+    List<ApplicationPrivilegeDescriptor> applicationPrivilegeDescriptors;
+
+    @Before
+    public void setup() {
+        applicationPrivilegeDescriptors = new ArrayList<>();
+    }
+
+    public void testRoleConstructorWithLimitedRole() {
+        Role fromRole = Role.builder("a-role").build();
+        Role limitedByRole = Role.builder("limited-role").build();
+        Role role = LimitedRole.createLimitedRole(fromRole, limitedByRole);
+        assertNotNull(role);
+
+        NullPointerException npe = expectThrows(NullPointerException.class, () -> LimitedRole.createLimitedRole(fromRole, null));
+        assertThat(npe.getMessage(), containsString("limited by role is required to create limited role"));
+    }
+
+    public void testAuthorize() {
+        IndexMetaData.Builder imbBuilder = IndexMetaData
+                .builder("_index").settings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                        .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1).put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
+                .putAlias(AliasMetaData.builder("_alias"));
+        IndexMetaData.Builder imbBuilder1 = IndexMetaData
+                .builder("_index1").settings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                        .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1).put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
+                .putAlias(AliasMetaData.builder("_alias1"));
+        MetaData md = MetaData.builder().put(imbBuilder).put(imbBuilder1).build();
+        FieldPermissionsCache fieldPermissionsCache = new FieldPermissionsCache(Settings.EMPTY);
+        Role fromRole = Role.builder("a-role").cluster(Collections.singleton(ClusterPrivilegeName.MANAGE_SECURITY), Collections.emptyList())
+                .add(IndexPrivilege.ALL, "_index").add(IndexPrivilege.CREATE_INDEX, "_index1").build();
+
+        IndicesAccessControl iac = fromRole.authorize(SearchAction.NAME, Sets.newHashSet("_index", "_alias1"), md, fieldPermissionsCache);
+        assertThat(iac.getIndexPermissions("_index"), is(notNullValue()));
+        assertThat(iac.getIndexPermissions("_index").isGranted(), is(true));
+        assertThat(iac.getIndexPermissions("_index1"), is(notNullValue()));
+        assertThat(iac.getIndexPermissions("_index1").isGranted(), is(false));
+        iac = fromRole.authorize(CreateIndexAction.NAME, Sets.newHashSet("_index", "_index1"), md, fieldPermissionsCache);
+        assertThat(iac.getIndexPermissions("_index"), is(notNullValue()));
+        assertThat(iac.getIndexPermissions("_index").isGranted(), is(true));
+        assertThat(iac.getIndexPermissions("_index1"), is(notNullValue()));
+        assertThat(iac.getIndexPermissions("_index1").isGranted(), is(true));
+
+        {
+            Role limitedByRole = Role.builder("limited-role")
+                    .cluster(Collections.singleton(ClusterPrivilegeName.ALL), Collections.emptyList()).add(IndexPrivilege.READ, "_index")
+                    .add(IndexPrivilege.NONE, "_index1").build();
+            iac = limitedByRole.authorize(SearchAction.NAME, Sets.newHashSet("_index", "_alias1"), md, fieldPermissionsCache);
+            assertThat(iac.getIndexPermissions("_index"), is(notNullValue()));
+            assertThat(iac.getIndexPermissions("_index").isGranted(), is(true));
+            assertThat(iac.getIndexPermissions("_index1"), is(notNullValue()));
+            assertThat(iac.getIndexPermissions("_index1").isGranted(), is(false));
+            iac = limitedByRole.authorize(DeleteIndexAction.NAME, Sets.newHashSet("_index", "_alias1"), md, fieldPermissionsCache);
+            assertThat(iac.getIndexPermissions("_index"), is(notNullValue()));
+            assertThat(iac.getIndexPermissions("_index").isGranted(), is(false));
+            assertThat(iac.getIndexPermissions("_index1"), is(notNullValue()));
+            assertThat(iac.getIndexPermissions("_index1").isGranted(), is(false));
+            iac = limitedByRole.authorize(CreateIndexAction.NAME, Sets.newHashSet("_index", "_alias1"), md, fieldPermissionsCache);
+            assertThat(iac.getIndexPermissions("_index"), is(notNullValue()));
+            assertThat(iac.getIndexPermissions("_index").isGranted(), is(false));
+            assertThat(iac.getIndexPermissions("_index1"), is(notNullValue()));
+            assertThat(iac.getIndexPermissions("_index1").isGranted(), is(false));
+
+            Role role = LimitedRole.createLimitedRole(fromRole, limitedByRole);
+            iac = role.authorize(SearchAction.NAME, Sets.newHashSet("_index", "_alias1"), md, fieldPermissionsCache);
+            assertThat(iac.getIndexPermissions("_index"), is(notNullValue()));
+            assertThat(iac.getIndexPermissions("_index").isGranted(), is(true));
+            assertThat(iac.getIndexPermissions("_index1"), is(notNullValue()));
+            assertThat(iac.getIndexPermissions("_index1").isGranted(), is(false));
+            iac = role.authorize(DeleteIndexAction.NAME, Sets.newHashSet("_index", "_alias1"), md, fieldPermissionsCache);
+            assertThat(iac.getIndexPermissions("_index"), is(notNullValue()));
+            assertThat(iac.getIndexPermissions("_index").isGranted(), is(false));
+            assertThat(iac.getIndexPermissions("_index1"), is(notNullValue()));
+            assertThat(iac.getIndexPermissions("_index1").isGranted(), is(false));
+            iac = role.authorize(CreateIndexAction.NAME, Sets.newHashSet("_index", "_index1"), md, fieldPermissionsCache);
+            assertThat(iac.getIndexPermissions("_index"), is(notNullValue()));
+            assertThat(iac.getIndexPermissions("_index").isGranted(), is(false));
+            assertThat(iac.getIndexPermissions("_index1"), is(notNullValue()));
+            assertThat(iac.getIndexPermissions("_index1").isGranted(), is(false));
+        }
+    }
+
+    public void testCheckClusterAction() {
+        Role fromRole = Role.builder("a-role").cluster(Collections.singleton(ClusterPrivilegeName.MANAGE_SECURITY), Collections.emptyList())
+                .build();
+        assertThat(fromRole.checkClusterAction("cluster:admin/xpack/security/x", mock(TransportRequest.class)), is(true));
+        {
+            Role limitedByRole = Role.builder("limited-role")
+                    .cluster(Collections.singleton(ClusterPrivilegeName.ALL), Collections.emptyList()).build();
+            assertThat(limitedByRole.checkClusterAction("cluster:admin/xpack/security/x", mock(TransportRequest.class)), is(true));
+            assertThat(limitedByRole.checkClusterAction("cluster:other-action", mock(TransportRequest.class)), is(true));
+            Role role = LimitedRole.createLimitedRole(fromRole, limitedByRole);
+            assertThat(role.checkClusterAction("cluster:admin/xpack/security/x", mock(TransportRequest.class)), is(true));
+            assertThat(role.checkClusterAction("cluster:other-action", mock(TransportRequest.class)), is(false));
+        }
+        {
+            Role limitedByRole = Role.builder("limited-role")
+                    .cluster(Collections.singleton(ClusterPrivilegeName.MONITOR), Collections.emptyList()).build();
+            assertThat(limitedByRole.checkClusterAction("cluster:monitor/me", mock(TransportRequest.class)), is(true));
+            Role role = LimitedRole.createLimitedRole(fromRole, limitedByRole);
+            assertThat(role.checkClusterAction("cluster:monitor/me", mock(TransportRequest.class)), is(false));
+            assertThat(role.checkClusterAction("cluster:admin/xpack/security/x", mock(TransportRequest.class)), is(false));
+        }
+    }
+
+    public void testCheckIndicesAction() {
+        Role fromRole = Role.builder("a-role").add(IndexPrivilege.READ, "ind-1").build();
+        assertThat(fromRole.checkIndicesAction(SearchAction.NAME), is(true));
+        assertThat(fromRole.checkIndicesAction(CreateIndexAction.NAME), is(false));
+
+        {
+            Role limitedByRole = Role.builder("limited-role").add(IndexPrivilege.ALL, "ind-1").build();
+            assertThat(limitedByRole.checkIndicesAction(SearchAction.NAME), is(true));
+            assertThat(limitedByRole.checkIndicesAction(CreateIndexAction.NAME), is(true));
+            Role role = LimitedRole.createLimitedRole(fromRole, limitedByRole);
+            assertThat(role.checkIndicesAction(SearchAction.NAME), is(true));
+            assertThat(role.checkIndicesAction(CreateIndexAction.NAME), is(false));
+        }
+        {
+            Role limitedByRole = Role.builder("limited-role").add(IndexPrivilege.NONE, "ind-1").build();
+            assertThat(limitedByRole.checkIndicesAction(SearchAction.NAME), is(false));
+            Role role = LimitedRole.createLimitedRole(fromRole, limitedByRole);
+            assertThat(role.checkIndicesAction(SearchAction.NAME), is(false));
+            assertThat(role.checkIndicesAction(CreateIndexAction.NAME), is(false));
+        }
+    }
+
+    public void testAllowedIndicesMatcher() {
+        Role fromRole = Role.builder("a-role").add(IndexPrivilege.READ, "ind-1*").build();
+        assertThat(fromRole.allowedIndicesMatcher(SearchAction.NAME).test("ind-1"), is(true));
+        assertThat(fromRole.allowedIndicesMatcher(SearchAction.NAME).test("ind-11"), is(true));
+        assertThat(fromRole.allowedIndicesMatcher(SearchAction.NAME).test("ind-2"), is(false));
+
+        {
+            Role limitedByRole = Role.builder("limited-role").add(IndexPrivilege.READ, "ind-1", "ind-2").build();
+            assertThat(limitedByRole.allowedIndicesMatcher(SearchAction.NAME).test("ind-1"), is(true));
+            assertThat(limitedByRole.allowedIndicesMatcher(SearchAction.NAME).test("ind-11"), is(false));
+            assertThat(limitedByRole.allowedIndicesMatcher(SearchAction.NAME).test("ind-2"), is(true));
+            Role role = LimitedRole.createLimitedRole(fromRole, limitedByRole);
+            assertThat(role.allowedIndicesMatcher(SearchAction.NAME).test("ind-1"), is(true));
+            assertThat(role.allowedIndicesMatcher(SearchAction.NAME).test("ind-11"), is(false));
+            assertThat(role.allowedIndicesMatcher(SearchAction.NAME).test("ind-2"), is(false));
+        }
+        {
+            Role limitedByRole = Role.builder("limited-role").add(IndexPrivilege.READ, "ind-*").build();
+            assertThat(limitedByRole.allowedIndicesMatcher(SearchAction.NAME).test("ind-1"), is(true));
+            assertThat(limitedByRole.allowedIndicesMatcher(SearchAction.NAME).test("ind-2"), is(true));
+            Role role = LimitedRole.createLimitedRole(fromRole, limitedByRole);
+            assertThat(role.allowedIndicesMatcher(SearchAction.NAME).test("ind-1"), is(true));
+            assertThat(role.allowedIndicesMatcher(SearchAction.NAME).test("ind-2"), is(false));
+        }
+    }
+
+    public void testCheckClusterPrivilege() {
+        Role fromRole = Role.builder("a-role").cluster(Collections.singleton(ClusterPrivilegeName.MANAGE_SECURITY), Collections.emptyList())
+                .build();
+        assertThat(fromRole.grants(ClusterPrivilege.ALL), is(false));
+        assertThat(fromRole.grants(ClusterPrivilege.MANAGE_SECURITY), is(true));
+
+        {
+            Role limitedByRole = Role.builder("scoped-role")
+                    .cluster(Collections.singleton(ClusterPrivilegeName.ALL), Collections.emptyList()).build();
+            assertThat(limitedByRole.grants(ClusterPrivilege.ALL), is(true));
+            assertThat(limitedByRole.grants(ClusterPrivilege.MANAGE_SECURITY), is(true));
+            Role role = LimitedRole.createLimitedRole(fromRole, limitedByRole);
+            assertThat(role.grants(ClusterPrivilege.ALL), is(false));
+            assertThat(role.grants(ClusterPrivilege.MANAGE_SECURITY), is(true));
+        }
+        {
+            Role limitedByRole = Role.builder("scoped-role")
+                    .cluster(Collections.singleton(ClusterPrivilegeName.MONITOR), Collections.emptyList()).build();
+            assertThat(limitedByRole.grants(ClusterPrivilege.ALL), is(false));
+            assertThat(limitedByRole.grants(ClusterPrivilege.MONITOR), is(true));
+            Role role = LimitedRole.createLimitedRole(fromRole, limitedByRole);
+            assertThat(role.grants(ClusterPrivilege.ALL), is(false));
+            assertThat(role.grants(ClusterPrivilege.MANAGE_SECURITY), is(false));
+            assertThat(role.grants(ClusterPrivilege.MONITOR), is(false));
+        }
+    }
+
+    public void testGetPrivilegesForIndexPatterns() {
+        Role fromRole = Role.builder("a-role").add(IndexPrivilege.READ, "ind-1*").build();
+        ResourcePrivilegesMap resourcePrivileges = fromRole.checkIndicesPrivileges(Collections.singleton("ind-1-1-*"), true,
+                Sets.newHashSet("read", "write"));
+        ResourcePrivilegesMap expectedAppPrivsByResource = new ResourcePrivilegesMap(false, Collections.singletonMap("ind-1-1-*",
+                ResourcePrivileges.builder("ind-1-1-*").addPrivilege("read", true).addPrivilege("write", false).build()));
+        verifyResourcesPrivileges(resourcePrivileges, expectedAppPrivsByResource);
+
+        resourcePrivileges = fromRole.checkIndicesPrivileges(Collections.singleton("ind-*"), true, Sets.newHashSet("read", "write"));
+        expectedAppPrivsByResource = new ResourcePrivilegesMap(false, Collections.singletonMap("ind-*",
+                ResourcePrivileges.builder("ind-*").addPrivilege("read", false).addPrivilege("write", false).build()));
+        verifyResourcesPrivileges(resourcePrivileges, expectedAppPrivsByResource);
+
+        {
+            Role limitedByRole = Role.builder("limited-role").add(IndexPrivilege.READ, "ind-1", "ind-2").build();
+            resourcePrivileges = limitedByRole.checkIndicesPrivileges(Collections.singleton("ind-1"), true, Collections.singleton("read"));
+            expectedAppPrivsByResource = new ResourcePrivilegesMap(true,
+                    Collections.singletonMap("ind-1", ResourcePrivileges.builder("ind-1").addPrivilege("read", true).build()));
+            verifyResourcesPrivileges(resourcePrivileges, expectedAppPrivsByResource);
+
+            resourcePrivileges = limitedByRole.checkIndicesPrivileges(Collections.singleton("ind-1-1-*"), true,
+                    Collections.singleton("read"));
+            expectedAppPrivsByResource = new ResourcePrivilegesMap(false,
+                    Collections.singletonMap("ind-1-1-*", ResourcePrivileges.builder("ind-1-1-*").addPrivilege("read", false).build()));
+            verifyResourcesPrivileges(resourcePrivileges, expectedAppPrivsByResource);
+
+            resourcePrivileges = limitedByRole.checkIndicesPrivileges(Collections.singleton("ind-*"), true, Collections.singleton("read"));
+            expectedAppPrivsByResource = new ResourcePrivilegesMap(false,
+                    Collections.singletonMap("ind-*", ResourcePrivileges.builder("ind-*").addPrivilege("read", false).build()));
+            verifyResourcesPrivileges(resourcePrivileges, expectedAppPrivsByResource);
+
+            Role role = LimitedRole.createLimitedRole(fromRole, limitedByRole);
+            resourcePrivileges = role.checkIndicesPrivileges(Collections.singleton("ind-1"), true, Collections.singleton("read"));
+            expectedAppPrivsByResource = new ResourcePrivilegesMap(true,
+                    Collections.singletonMap("ind-1", ResourcePrivileges.builder("ind-1").addPrivilege("read", true).build()));
+            verifyResourcesPrivileges(resourcePrivileges, expectedAppPrivsByResource);
+
+            resourcePrivileges = role.checkIndicesPrivileges(Sets.newHashSet("ind-1-1-*", "ind-1"), true, Collections.singleton("read"));
+            expectedAppPrivsByResource = new ResourcePrivilegesMap(false,
+                    mapBuilder().put("ind-1-1-*", ResourcePrivileges.builder("ind-1-1-*").addPrivilege("read", false).build())
+                            .put("ind-1", ResourcePrivileges.builder("ind-1").addPrivilege("read", true).build()).map());
+            verifyResourcesPrivileges(resourcePrivileges, expectedAppPrivsByResource);
+        }
+        {
+            fromRole = Role.builder("a-role")
+                    .add(FieldPermissions.DEFAULT, Collections.emptySet(), IndexPrivilege.READ, true, "ind-1*", ".security").build();
+            resourcePrivileges = fromRole.checkIndicesPrivileges(Sets.newHashSet("ind-1", ".security"), true,
+                    Collections.singleton("read"));
+            // Map<String, ResourcePrivileges> expectedResourceToResourcePrivs = new HashMap<>();
+            ;
+            expectedAppPrivsByResource = new ResourcePrivilegesMap(true,
+                    mapBuilder().put("ind-1", ResourcePrivileges.builder("ind-1").addPrivilege("read", true).build())
+                            .put(".security", ResourcePrivileges.builder(".security").addPrivilege("read", true).build()).map());
+            verifyResourcesPrivileges(resourcePrivileges, expectedAppPrivsByResource);
+
+            Role limitedByRole = Role.builder("limited-role").add(IndexPrivilege.READ, "ind-1", "ind-2").build();
+            resourcePrivileges = limitedByRole.checkIndicesPrivileges(Sets.newHashSet("ind-1", "ind-2", ".security"), true,
+                    Collections.singleton("read"));
+
+            expectedAppPrivsByResource = new ResourcePrivilegesMap(false,
+                    mapBuilder().put("ind-1", ResourcePrivileges.builder("ind-1").addPrivilege("read", true).build())
+                            .put("ind-2", ResourcePrivileges.builder("ind-2").addPrivilege("read", true).build())
+                            .put(".security", ResourcePrivileges.builder(".security").addPrivilege("read", false).build()).map());
+            verifyResourcesPrivileges(resourcePrivileges, expectedAppPrivsByResource);
+
+            Role role = LimitedRole.createLimitedRole(fromRole, limitedByRole);
+            resourcePrivileges = role.checkIndicesPrivileges(Sets.newHashSet("ind-1", "ind-2", ".security"), true,
+                    Collections.singleton("read"));
+
+            expectedAppPrivsByResource = new ResourcePrivilegesMap(false,
+                    mapBuilder().put("ind-1", ResourcePrivileges.builder("ind-1").addPrivilege("read", true).build())
+                            .put("ind-2", ResourcePrivileges.builder("ind-2").addPrivilege("read", false).build())
+                            .put(".security", ResourcePrivileges.builder(".security").addPrivilege("read", false).build()).map());
+            verifyResourcesPrivileges(resourcePrivileges, expectedAppPrivsByResource);
+        }
+    }
+
+    public void testGetApplicationPrivilegesByResource() {
+        final ApplicationPrivilege app1Read = defineApplicationPrivilege("app1", "read", "data:read/*");
+        final ApplicationPrivilege app1All = defineApplicationPrivilege("app1", "all", "*");
+        final ApplicationPrivilege app2Read = defineApplicationPrivilege("app2", "read", "data:read/*");
+        final ApplicationPrivilege app2Write = defineApplicationPrivilege("app2", "write", "data:write/*");
+
+        Role fromRole = Role.builder("test-role").addApplicationPrivilege(app1Read, Collections.singleton("foo/*"))
+                .addApplicationPrivilege(app1All, Collections.singleton("foo/bar/baz"))
+                .addApplicationPrivilege(app2Read, Collections.singleton("foo/bar/*"))
+                .addApplicationPrivilege(app2Write, Collections.singleton("*/bar/*")).build();
+
+        Set<String> forPrivilegeNames = Sets.newHashSet("read", "write", "all");
+        ResourcePrivilegesMap appPrivsByResource = fromRole.checkApplicationResourcePrivileges("app1", Collections.singleton("*"),
+                forPrivilegeNames, applicationPrivilegeDescriptors);
+        ResourcePrivilegesMap expectedAppPrivsByResource = new ResourcePrivilegesMap(false, Collections.singletonMap("*", ResourcePrivileges
+                .builder("*").addPrivilege("read", false).addPrivilege("write", false).addPrivilege("all", false).build()));
+        verifyResourcesPrivileges(appPrivsByResource, expectedAppPrivsByResource);
+
+        appPrivsByResource = fromRole.checkApplicationResourcePrivileges("app1", Collections.singleton("foo/x/y"), forPrivilegeNames,
+                applicationPrivilegeDescriptors);
+        expectedAppPrivsByResource = new ResourcePrivilegesMap(false, Collections.singletonMap("foo/x/y", ResourcePrivileges
+                .builder("foo/x/y").addPrivilege("read", true).addPrivilege("write", false).addPrivilege("all", false).build()));
+        verifyResourcesPrivileges(appPrivsByResource, expectedAppPrivsByResource);
+
+        appPrivsByResource = fromRole.checkApplicationResourcePrivileges("app2", Collections.singleton("foo/bar/a"), forPrivilegeNames,
+                applicationPrivilegeDescriptors);
+        expectedAppPrivsByResource = new ResourcePrivilegesMap(false, Collections.singletonMap("foo/bar/a", ResourcePrivileges
+                .builder("foo/bar/a").addPrivilege("read", true).addPrivilege("write", true).addPrivilege("all", false).build()));
+        verifyResourcesPrivileges(appPrivsByResource, expectedAppPrivsByResource);
+
+        appPrivsByResource = fromRole.checkApplicationResourcePrivileges("app2", Collections.singleton("moon/bar/a"), forPrivilegeNames,
+                applicationPrivilegeDescriptors);
+        expectedAppPrivsByResource = new ResourcePrivilegesMap(false, Collections.singletonMap("moon/bar/a", ResourcePrivileges
+                .builder("moon/bar/a").addPrivilege("read", false).addPrivilege("write", true).addPrivilege("all", false).build()));
+        verifyResourcesPrivileges(appPrivsByResource, expectedAppPrivsByResource);
+
+        {
+            Role limitedByRole = Role.builder("test-role-scoped").addApplicationPrivilege(app1Read, Collections.singleton("foo/scoped/*"))
+                    .addApplicationPrivilege(app2Read, Collections.singleton("foo/bar/*"))
+                    .addApplicationPrivilege(app2Write, Collections.singleton("moo/bar/*")).build();
+            appPrivsByResource = limitedByRole.checkApplicationResourcePrivileges("app1", Collections.singleton("*"), forPrivilegeNames,
+                    applicationPrivilegeDescriptors);
+            expectedAppPrivsByResource = new ResourcePrivilegesMap(false, Collections.singletonMap("*", ResourcePrivileges.builder("*")
+                    .addPrivilege("read", false).addPrivilege("write", false).addPrivilege("all", false).build()));
+            verifyResourcesPrivileges(appPrivsByResource, expectedAppPrivsByResource);
+
+            appPrivsByResource = limitedByRole.checkApplicationResourcePrivileges("app1", Collections.singleton("foo/x/y"),
+                    forPrivilegeNames, applicationPrivilegeDescriptors);
+            expectedAppPrivsByResource = new ResourcePrivilegesMap(false, Collections.singletonMap("foo/x/y", ResourcePrivileges
+                    .builder("foo/x/y").addPrivilege("read", false).addPrivilege("write", false).addPrivilege("all", false).build()));
+            verifyResourcesPrivileges(appPrivsByResource, expectedAppPrivsByResource);
+
+            appPrivsByResource = limitedByRole.checkApplicationResourcePrivileges("app2", Collections.singleton("foo/bar/a"),
+                    forPrivilegeNames, applicationPrivilegeDescriptors);
+            expectedAppPrivsByResource = new ResourcePrivilegesMap(false, Collections.singletonMap("foo/bar/a", ResourcePrivileges
+                    .builder("foo/bar/a").addPrivilege("read", true).addPrivilege("write", false).addPrivilege("all", false).build()));
+            verifyResourcesPrivileges(appPrivsByResource, expectedAppPrivsByResource);
+
+            appPrivsByResource = limitedByRole.checkApplicationResourcePrivileges("app2", Collections.singleton("moon/bar/a"),
+                    forPrivilegeNames, applicationPrivilegeDescriptors);
+            expectedAppPrivsByResource = new ResourcePrivilegesMap(false, Collections.singletonMap("moon/bar/a", ResourcePrivileges
+                    .builder("moon/bar/a").addPrivilege("read", false).addPrivilege("write", false).addPrivilege("all", false).build()));
+            verifyResourcesPrivileges(appPrivsByResource, expectedAppPrivsByResource);
+
+            Role role = LimitedRole.createLimitedRole(fromRole, limitedByRole);
+            appPrivsByResource = role.checkApplicationResourcePrivileges("app2", Collections.singleton("foo/bar/a"), forPrivilegeNames,
+                    applicationPrivilegeDescriptors);
+            expectedAppPrivsByResource = new ResourcePrivilegesMap(false, Collections.singletonMap("foo/bar/a", ResourcePrivileges
+                    .builder("foo/bar/a").addPrivilege("read", true).addPrivilege("write", false).addPrivilege("all", false).build()));
+            verifyResourcesPrivileges(appPrivsByResource, expectedAppPrivsByResource);
+
+            appPrivsByResource = role.checkApplicationResourcePrivileges("app2", Collections.singleton("moon/bar/a"), forPrivilegeNames,
+                    applicationPrivilegeDescriptors);
+            expectedAppPrivsByResource = new ResourcePrivilegesMap(false, Collections.singletonMap("moon/bar/a", ResourcePrivileges
+                    .builder("moon/bar/a").addPrivilege("read", false).addPrivilege("write", false).addPrivilege("all", false).build()));
+            verifyResourcesPrivileges(appPrivsByResource, expectedAppPrivsByResource);
+
+            appPrivsByResource = role.checkApplicationResourcePrivileges("unknown", Collections.singleton("moon/bar/a"), forPrivilegeNames,
+                    applicationPrivilegeDescriptors);
+            expectedAppPrivsByResource = new ResourcePrivilegesMap(false, Collections.singletonMap("moon/bar/a", ResourcePrivileges
+                    .builder("moon/bar/a").addPrivilege("read", false).addPrivilege("write", false).addPrivilege("all", false).build()));
+            verifyResourcesPrivileges(appPrivsByResource, expectedAppPrivsByResource);
+
+            appPrivsByResource = role.checkApplicationResourcePrivileges("app2", Collections.singleton("moo/bar/a"),
+                    Sets.newHashSet("read", "write", "all", "unknown"), applicationPrivilegeDescriptors);
+            expectedAppPrivsByResource = new ResourcePrivilegesMap(false,
+                    Collections.singletonMap("moo/bar/a", ResourcePrivileges.builder("moo/bar/a").addPrivilege("read", false)
+                            .addPrivilege("write", true).addPrivilege("all", false).addPrivilege("unknown", false).build()));
+            verifyResourcesPrivileges(appPrivsByResource, expectedAppPrivsByResource);
+        }
+    }
+
+    private void verifyResourcesPrivileges(ResourcePrivilegesMap resourcePrivileges, ResourcePrivilegesMap expectedAppPrivsByResource) {
+        assertThat(resourcePrivileges, equalTo(expectedAppPrivsByResource));
+    }
+
+    private ApplicationPrivilege defineApplicationPrivilege(String app, String name, String... actions) {
+        applicationPrivilegeDescriptors
+                .add(new ApplicationPrivilegeDescriptor(app, name, Sets.newHashSet(actions), Collections.emptyMap()));
+        return new ApplicationPrivilege(app, name, actions);
+    }
+
+    private static MapBuilder<String, ResourcePrivileges> mapBuilder() {
+        return MapBuilder.newMapBuilder();
+    }
+
+}

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/permission/ResourcePrivilegesMapTests.java

@@ -0,0 +1,91 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.authz.permission;
+
+import org.elasticsearch.common.collect.MapBuilder;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
+
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.nullValue;
+
+public class ResourcePrivilegesMapTests extends ESTestCase {
+
+    public void testBuilder() {
+        ResourcePrivilegesMap instance = ResourcePrivilegesMap.builder()
+                .addResourcePrivilege("*", mapBuilder().put("read", true).put("write", true).map()).build();
+        assertThat(instance.allAllowed(), is(true));
+        assertThat(instance.getResourceToResourcePrivileges().size(), is(1));
+        assertThat(instance.getResourceToResourcePrivileges().get("*").isAllowed("read"), is(true));
+        assertThat(instance.getResourceToResourcePrivileges().get("*").isAllowed("write"), is(true));
+
+        instance = ResourcePrivilegesMap.builder().addResourcePrivilege("*", mapBuilder().put("read", true).put("write", false).map())
+                .build();
+        assertThat(instance.allAllowed(), is(false));
+        assertThat(instance.getResourceToResourcePrivileges().size(), is(1));
+        assertThat(instance.getResourceToResourcePrivileges().get("*").isAllowed("read"), is(true));
+        assertThat(instance.getResourceToResourcePrivileges().get("*").isAllowed("write"), is(false));
+
+        instance = ResourcePrivilegesMap.builder()
+                .addResourcePrivilege("some-other", mapBuilder().put("index", true).put("write", true).map())
+                .addResourcePrivilegesMap(instance).build();
+        assertThat(instance.allAllowed(), is(false));
+        assertThat(instance.getResourceToResourcePrivileges().size(), is(2));
+        assertThat(instance.getResourceToResourcePrivileges().get("*").isAllowed("read"), is(true));
+        assertThat(instance.getResourceToResourcePrivileges().get("*").isAllowed("write"), is(false));
+        assertThat(instance.getResourceToResourcePrivileges().get("some-other").isAllowed("index"), is(true));
+        assertThat(instance.getResourceToResourcePrivileges().get("some-other").isAllowed("write"), is(true));
+    }
+
+    public void testIntersection() {
+        ResourcePrivilegesMap instance = ResourcePrivilegesMap.builder()
+                .addResourcePrivilege("*", mapBuilder().put("read", true).put("write", true).map())
+                .addResourcePrivilege("index-*", mapBuilder().put("read", true).put("write", true).map()).build();
+        ResourcePrivilegesMap otherInstance = ResourcePrivilegesMap.builder()
+                .addResourcePrivilege("*", mapBuilder().put("read", true).put("write", false).map())
+                .addResourcePrivilege("index-*", mapBuilder().put("read", false).put("write", true).map())
+                .addResourcePrivilege("index-uncommon", mapBuilder().put("read", false).put("write", true).map()).build();
+        ResourcePrivilegesMap result = ResourcePrivilegesMap.intersection(instance, otherInstance);
+        assertThat(result.allAllowed(), is(false));
+        assertThat(result.getResourceToResourcePrivileges().size(), is(2));
+        assertThat(result.getResourceToResourcePrivileges().get("*").isAllowed("read"), is(true));
+        assertThat(result.getResourceToResourcePrivileges().get("*").isAllowed("write"), is(false));
+        assertThat(result.getResourceToResourcePrivileges().get("index-*").isAllowed("read"), is(false));
+        assertThat(result.getResourceToResourcePrivileges().get("index-*").isAllowed("write"), is(true));
+        assertThat(result.getResourceToResourcePrivileges().get("index-uncommon"), is(nullValue()));
+    }
+
+    public void testEqualsHashCode() {
+        ResourcePrivilegesMap instance = ResourcePrivilegesMap.builder()
+                .addResourcePrivilege("*", mapBuilder().put("read", true).put("write", true).map()).build();
+
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(instance, (original) -> {
+            return ResourcePrivilegesMap.builder().addResourcePrivilegesMap(original).build();
+        });
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(instance, (original) -> {
+            return ResourcePrivilegesMap.builder().addResourcePrivilegesMap(original).build();
+        }, ResourcePrivilegesMapTests::mutateTestItem);
+    }
+
+    private static ResourcePrivilegesMap mutateTestItem(ResourcePrivilegesMap original) {
+        switch (randomIntBetween(0, 1)) {
+        case 0:
+            return ResourcePrivilegesMap.builder()
+                    .addResourcePrivilege(randomAlphaOfLength(6), mapBuilder().put("read", true).put("write", true).map()).build();
+        case 1:
+            return ResourcePrivilegesMap.builder().addResourcePrivilege("*", mapBuilder().put("read", false).put("write", false).map())
+                    .build();
+        default:
+            return ResourcePrivilegesMap.builder()
+                    .addResourcePrivilege(randomAlphaOfLength(6), mapBuilder().put("read", true).put("write", true).map()).build();
+        }
+    }
+
+    private static MapBuilder<String, Boolean> mapBuilder() {
+        return MapBuilder.newMapBuilder();
+    }
+}

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/permission/ResourcePrivilegesTests.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.authz.permission;
+
+import org.elasticsearch.common.collect.MapBuilder;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
+
+import java.util.Collections;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+
+public class ResourcePrivilegesTests extends ESTestCase {
+
+    public void testBuilder() {
+        ResourcePrivileges instance = createInstance();
+        ResourcePrivileges expected = new ResourcePrivileges("*", mapBuilder().put("read", true).put("write", false).map());
+        assertThat(instance, equalTo(expected));
+    }
+
+    public void testWhenSamePrivilegeExists() {
+        ResourcePrivileges.Builder builder = ResourcePrivileges.builder("*").addPrivilege("read", true);
+
+        Map<String, Boolean> mapWhereReadIsAllowed = mapBuilder().put("read", true).map();
+        builder.addPrivileges(mapWhereReadIsAllowed);
+        assertThat(builder.build().isAllowed("read"), is(true));
+
+        Map<String, Boolean> mapWhereReadIsDenied = mapBuilder().put("read", false).map();
+        builder.addPrivileges(mapWhereReadIsDenied);
+        assertThat(builder.build().isAllowed("read"), is(false));
+    }
+
+    public void testEqualsHashCode() {
+        ResourcePrivileges instance = createInstance();
+
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(instance, (original) -> {
+            return ResourcePrivileges.builder(original.getResource()).addPrivileges(original.getPrivileges()).build();
+        });
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(instance, (original) -> {
+            return ResourcePrivileges.builder(original.getResource()).addPrivileges(original.getPrivileges()).build();
+        }, ResourcePrivilegesTests::mutateTestItem);
+    }
+
+    private ResourcePrivileges createInstance() {
+        ResourcePrivileges instance = ResourcePrivileges.builder("*").addPrivilege("read", true)
+                .addPrivileges(Collections.singletonMap("write", false)).build();
+        return instance;
+    }
+
+    private static ResourcePrivileges mutateTestItem(ResourcePrivileges original) {
+        switch (randomIntBetween(0, 1)) {
+        case 0:
+            return ResourcePrivileges.builder(randomAlphaOfLength(6)).addPrivileges(original.getPrivileges()).build();
+        case 1:
+            return ResourcePrivileges.builder(original.getResource()).addPrivileges(Collections.emptyMap()).build();
+        default:
+            return ResourcePrivileges.builder(randomAlphaOfLength(6)).addPrivileges(Collections.emptyMap()).build();
+        }
+    }
+
+    private static MapBuilder<String, Boolean> mapBuilder() {
+        return MapBuilder.newMapBuilder();
+    }
+}

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/support/SecurityQueryTemplateEvaluatorTests.java

@@ -0,0 +1,94 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.authz.support;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.query.TermQueryBuilder;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
+import org.elasticsearch.script.TemplateScript;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.security.user.User;
+import org.junit.Before;
+import org.mockito.ArgumentCaptor;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.sameInstance;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyZeroInteractions;
+import static org.mockito.Mockito.when;
+
+public class SecurityQueryTemplateEvaluatorTests extends ESTestCase {
+    private ScriptService scriptService;
+
+    @Before
+    public void setup() throws Exception {
+        scriptService = mock(ScriptService.class);
+    }
+
+    public void testTemplating() throws Exception {
+        User user = new User("_username", new String[] { "role1", "role2" }, "_full_name", "_email",
+                Collections.singletonMap("key", "value"), true);
+
+        TemplateScript.Factory compiledTemplate = templateParams -> new TemplateScript(templateParams) {
+            @Override
+            public String execute() {
+                return "rendered_text";
+            }
+        };
+
+        when(scriptService.compile(any(Script.class), eq(TemplateScript.CONTEXT))).thenReturn(compiledTemplate);
+
+        XContentBuilder builder = jsonBuilder();
+        String query = Strings.toString(new TermQueryBuilder("field", "{{_user.username}}").toXContent(builder, ToXContent.EMPTY_PARAMS));
+        Script script = new Script(ScriptType.INLINE, "mustache", query, Collections.singletonMap("custom", "value"));
+        builder = jsonBuilder().startObject().field("template");
+        script.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        String querySource = Strings.toString(builder.endObject());
+
+        SecurityQueryTemplateEvaluator.evaluateTemplate(querySource, scriptService, user);
+        ArgumentCaptor<Script> argument = ArgumentCaptor.forClass(Script.class);
+        verify(scriptService).compile(argument.capture(), eq(TemplateScript.CONTEXT));
+        Script usedScript = argument.getValue();
+        assertThat(usedScript.getIdOrCode(), equalTo(script.getIdOrCode()));
+        assertThat(usedScript.getType(), equalTo(script.getType()));
+        assertThat(usedScript.getLang(), equalTo("mustache"));
+        assertThat(usedScript.getOptions(), equalTo(script.getOptions()));
+        assertThat(usedScript.getParams().size(), equalTo(2));
+        assertThat(usedScript.getParams().get("custom"), equalTo("value"));
+
+        Map<String, Object> userModel = new HashMap<>();
+        userModel.put("username", user.principal());
+        userModel.put("full_name", user.fullName());
+        userModel.put("email", user.email());
+        userModel.put("roles", Arrays.asList(user.roles()));
+        userModel.put("metadata", user.metadata());
+        assertThat(usedScript.getParams().get("_user"), equalTo(userModel));
+
+    }
+
+    public void testSkipTemplating() throws Exception {
+        XContentBuilder builder = jsonBuilder();
+        String querySource = Strings.toString(new TermQueryBuilder("field", "value").toXContent(builder, ToXContent.EMPTY_PARAMS));
+        String result = SecurityQueryTemplateEvaluator.evaluateTemplate(querySource, scriptService, null);
+        assertThat(result, sameInstance(querySource));
+        verifyZeroInteractions(scriptService);
+    }
+
+}

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java

@@ -34,8 +34,8 @@ import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
 import org.elasticsearch.xpack.core.ml.action.PutDatafeedAction;
-import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.core.rollup.action.GetRollupIndexCapsAction;
 import org.elasticsearch.xpack.core.rollup.action.RollupSearchAction;
 import org.elasticsearch.xpack.core.security.SecurityContext;
@@ -43,6 +43,7 @@ import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesAction;
 import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesRequest;
 import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesResponse;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.permission.ResourcePrivileges;
 import org.elasticsearch.xpack.core.security.support.Exceptions;
 import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider;
 import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider;
@@ -149,7 +150,7 @@ public class TransportPutDatafeedAction extends TransportMasterNodeAction<PutDat
         } else {
             XContentBuilder builder = JsonXContent.contentBuilder();
             builder.startObject();
-            for (HasPrivilegesResponse.ResourcePrivileges index : response.getIndexPrivileges()) {
+            for (ResourcePrivileges index : response.getIndexPrivileges()) {
                 builder.field(index.getResource());
                 builder.map(index.getPrivileges());
             }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ApiKeySSLBootstrapCheck.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security;
+
+import org.elasticsearch.bootstrap.BootstrapCheck;
+import org.elasticsearch.bootstrap.BootstrapContext;
+import org.elasticsearch.xpack.core.XPackSettings;
+
+import java.util.Locale;
+
+/**
+ * Bootstrap check to ensure that the user has enabled HTTPS when using the api key service
+ */
+public final class ApiKeySSLBootstrapCheck implements BootstrapCheck {
+
+    @Override
+    public BootstrapCheckResult check(BootstrapContext context) {
+        final Boolean httpsEnabled = XPackSettings.HTTP_SSL_ENABLED.get(context.settings());
+        final Boolean apiKeyServiceEnabled = XPackSettings.API_KEY_SERVICE_ENABLED_SETTING.get(context.settings());
+        if (httpsEnabled == false && apiKeyServiceEnabled) {
+            final String message = String.format(
+                    Locale.ROOT,
+                    "HTTPS is required in order to use the API key service; "
+                            + "please enable HTTPS using the [%s] setting or disable the API key service using the [%s] setting",
+                    XPackSettings.HTTP_SSL_ENABLED.getKey(),
+                    XPackSettings.API_KEY_SERVICE_ENABLED_SETTING.getKey());
+            return BootstrapCheckResult.failure(message);
+        }
+        return BootstrapCheckResult.success();
+    }
+
+
+}

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java

@@ -77,6 +77,9 @@ import org.elasticsearch.xpack.core.security.SecurityContext;
 import org.elasticsearch.xpack.core.security.SecurityExtension;
 import org.elasticsearch.xpack.core.security.SecurityField;
 import org.elasticsearch.xpack.core.security.SecuritySettings;
+import org.elasticsearch.xpack.core.security.action.CreateApiKeyAction;
+import org.elasticsearch.xpack.core.security.action.GetApiKeyAction;
+import org.elasticsearch.xpack.core.security.action.InvalidateApiKeyAction;
 import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesAction;
 import org.elasticsearch.xpack.core.security.action.privilege.GetPrivilegesAction;
 import org.elasticsearch.xpack.core.security.action.privilege.PutPrivilegesAction;
@@ -127,6 +130,9 @@ import org.elasticsearch.xpack.core.ssl.TLSLicenseBootstrapCheck;
 import org.elasticsearch.xpack.core.ssl.action.GetCertificateInfoAction;
 import org.elasticsearch.xpack.core.ssl.action.TransportGetCertificateInfoAction;
 import org.elasticsearch.xpack.core.ssl.rest.RestGetCertificateInfoAction;
+import org.elasticsearch.xpack.security.action.TransportCreateApiKeyAction;
+import org.elasticsearch.xpack.security.action.TransportGetApiKeyAction;
+import org.elasticsearch.xpack.security.action.TransportInvalidateApiKeyAction;
 import org.elasticsearch.xpack.security.action.filter.SecurityActionFilter;
 import org.elasticsearch.xpack.security.action.interceptor.BulkShardRequestInterceptor;
 import org.elasticsearch.xpack.security.action.interceptor.IndicesAliasesRequestInterceptor;
@@ -163,6 +169,7 @@ import org.elasticsearch.xpack.security.action.user.TransportSetEnabledAction;
 import org.elasticsearch.xpack.security.audit.AuditTrail;
 import org.elasticsearch.xpack.security.audit.AuditTrailService;
 import org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail;
+import org.elasticsearch.xpack.security.authc.ApiKeyService;
 import org.elasticsearch.xpack.security.authc.AuthenticationService;
 import org.elasticsearch.xpack.security.authc.InternalRealms;
 import org.elasticsearch.xpack.security.authc.Realms;
@@ -180,6 +187,9 @@ import org.elasticsearch.xpack.security.authz.store.NativeRolesStore;
 import org.elasticsearch.xpack.security.ingest.SetSecurityUserProcessor;
 import org.elasticsearch.xpack.security.rest.SecurityRestFilter;
 import org.elasticsearch.xpack.security.rest.action.RestAuthenticateAction;
+import org.elasticsearch.xpack.security.rest.action.RestCreateApiKeyAction;
+import org.elasticsearch.xpack.security.rest.action.RestGetApiKeyAction;
+import org.elasticsearch.xpack.security.rest.action.RestInvalidateApiKeyAction;
 import org.elasticsearch.xpack.security.rest.action.oauth2.RestGetTokenAction;
 import org.elasticsearch.xpack.security.rest.action.oauth2.RestInvalidateTokenAction;
 import org.elasticsearch.xpack.security.rest.action.privilege.RestDeletePrivilegesAction;
@@ -235,6 +245,7 @@ import java.util.stream.Collectors;
 import static java.util.Collections.emptyList;
 import static java.util.Collections.singletonList;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_FORMAT_SETTING;
+import static org.elasticsearch.xpack.core.XPackSettings.API_KEY_SERVICE_ENABLED_SETTING;
 import static org.elasticsearch.xpack.core.XPackSettings.HTTP_SSL_ENABLED;
 import static org.elasticsearch.xpack.security.support.SecurityIndexManager.INTERNAL_INDEX_FORMAT;
 import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME;
@@ -280,6 +291,7 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
             // fetched
             final List<BootstrapCheck> checks = new ArrayList<>();
             checks.addAll(Arrays.asList(
+                new ApiKeySSLBootstrapCheck(),
                 new TokenSSLBootstrapCheck(),
                 new PkiRealmBootstrapCheck(getSslService()),
                 new TLSLicenseBootstrapCheck(),
@@ -413,16 +425,10 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
 
         securityIndex.get().addIndexStateListener(nativeRoleMappingStore::onSecurityIndexStateChange);
 
-        final AuthenticationFailureHandler failureHandler = createAuthenticationFailureHandler(realms);
-
-        authcService.set(new AuthenticationService(settings, realms, auditTrailService, failureHandler, threadPool,
-                anonymousUser, tokenService));
-        components.add(authcService.get());
-        securityIndex.get().addIndexStateListener(authcService.get()::onSecurityIndexStateChange);
-
         final NativePrivilegeStore privilegeStore = new NativePrivilegeStore(settings, client, securityIndex.get());
         components.add(privilegeStore);
 
+        final FieldPermissionsCache fieldPermissionsCache = new FieldPermissionsCache(settings);
         final FileRolesStore fileRolesStore = new FileRolesStore(settings, env, resourceWatcherService, getLicenseState());
         final NativeRolesStore nativeRolesStore = new NativeRolesStore(settings, client, getLicenseState(), securityIndex.get());
         final ReservedRolesStore reservedRolesStore = new ReservedRolesStore();
@@ -431,13 +437,24 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
             rolesProviders.addAll(extension.getRolesProviders(settings, resourceWatcherService));
         }
         final CompositeRolesStore allRolesStore = new CompositeRolesStore(settings, fileRolesStore, nativeRolesStore,
-            reservedRolesStore, privilegeStore, rolesProviders, threadPool.getThreadContext(), getLicenseState());
+            reservedRolesStore, privilegeStore, rolesProviders, threadPool.getThreadContext(), getLicenseState(), fieldPermissionsCache);
         securityIndex.get().addIndexStateListener(allRolesStore::onSecurityIndexStateChange);
         // to keep things simple, just invalidate all cached entries on license change. this happens so rarely that the impact should be
         // minimal
         getLicenseState().addListener(allRolesStore::invalidateAll);
+
+        final ApiKeyService apiKeyService = new ApiKeyService(settings, Clock.systemUTC(), client, securityIndex.get(), clusterService,
+                allRolesStore);
+        components.add(apiKeyService);
+
+        final AuthenticationFailureHandler failureHandler = createAuthenticationFailureHandler(realms);
+        authcService.set(new AuthenticationService(settings, realms, auditTrailService, failureHandler, threadPool,
+                anonymousUser, tokenService, apiKeyService));
+        components.add(authcService.get());
+        securityIndex.get().addIndexStateListener(authcService.get()::onSecurityIndexStateChange);
+
         final AuthorizationService authzService = new AuthorizationService(settings, allRolesStore, clusterService,
-            auditTrailService, failureHandler, threadPool, anonymousUser);
+            auditTrailService, failureHandler, threadPool, anonymousUser, apiKeyService, fieldPermissionsCache);
         components.add(nativeRolesStore); // used by roles actions
         components.add(reservedRolesStore); // used by roles actions
         components.add(allRolesStore); // for SecurityFeatureSet and clear roles cache
@@ -499,6 +516,13 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
                     defaultFailureResponseHeaders.get("WWW-Authenticate").add(bearerScheme);
                 }
             }
+            if (API_KEY_SERVICE_ENABLED_SETTING.get(settings)) {
+                final String apiKeyScheme = "ApiKey";
+                if (defaultFailureResponseHeaders.computeIfAbsent("WWW-Authenticate", x -> new ArrayList<>())
+                    .contains(apiKeyScheme) == false) {
+                    defaultFailureResponseHeaders.get("WWW-Authenticate").add(apiKeyScheme);
+                }
+            }
             failureHandler = new DefaultAuthenticationFailureHandler(defaultFailureResponseHeaders);
         } else {
             logger.debug("Using authentication failure handler from extension [" + extensionName + "]");
@@ -583,6 +607,9 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
         settingsList.add(TokenService.DELETE_TIMEOUT);
         settingsList.add(SecurityServerTransportInterceptor.TRANSPORT_TYPE_PROFILE_SETTING);
         settingsList.addAll(SSLConfigurationSettings.getProfileSettings());
+        settingsList.add(ApiKeyService.PASSWORD_HASHING_ALGORITHM);
+        settingsList.add(ApiKeyService.DELETE_TIMEOUT);
+        settingsList.add(ApiKeyService.DELETE_INTERVAL);
 
         // hide settings
         settingsList.add(Setting.listSetting(SecurityField.setting("hide_settings"), Collections.emptyList(), Function.identity(),
@@ -686,7 +713,10 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
                 new ActionHandler<>(SamlInvalidateSessionAction.INSTANCE, TransportSamlInvalidateSessionAction.class),
                 new ActionHandler<>(GetPrivilegesAction.INSTANCE, TransportGetPrivilegesAction.class),
                 new ActionHandler<>(PutPrivilegesAction.INSTANCE, TransportPutPrivilegesAction.class),
-                new ActionHandler<>(DeletePrivilegesAction.INSTANCE, TransportDeletePrivilegesAction.class)
+                new ActionHandler<>(DeletePrivilegesAction.INSTANCE, TransportDeletePrivilegesAction.class),
+                new ActionHandler<>(CreateApiKeyAction.INSTANCE, TransportCreateApiKeyAction.class),
+                new ActionHandler<>(InvalidateApiKeyAction.INSTANCE, TransportInvalidateApiKeyAction.class),
+                new ActionHandler<>(GetApiKeyAction.INSTANCE, TransportGetApiKeyAction.class)
         );
     }
 
@@ -735,7 +765,10 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
                 new RestSamlInvalidateSessionAction(settings, restController, getLicenseState()),
                 new RestGetPrivilegesAction(settings, restController, getLicenseState()),
                 new RestPutPrivilegesAction(settings, restController, getLicenseState()),
-                new RestDeletePrivilegesAction(settings, restController, getLicenseState())
+                new RestDeletePrivilegesAction(settings, restController, getLicenseState()),
+                new RestCreateApiKeyAction(settings, restController, getLicenseState()),
+                new RestInvalidateApiKeyAction(settings, restController, getLicenseState()),
+                new RestGetApiKeyAction(settings, restController, getLicenseState())
         );
     }
 
@@ -887,7 +920,7 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
                     throw new IllegalStateException("unexpected call to getFieldFilter for index [" + index + "] which is not granted");
                 }
                 FieldPermissions fieldPermissions = indexPermissions.getFieldPermissions();
-                if (fieldPermissions == null) {
+                if (fieldPermissions.hasFieldLevelSecurity() == false) {
                     return MapperPlugin.NOOP_FIELD_PREDICATE;
                 }
                 return fieldPermissions::grantsAccessTo;

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportCreateApiKeyAction.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security.action;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.security.SecurityContext;
+import org.elasticsearch.xpack.core.security.action.CreateApiKeyAction;
+import org.elasticsearch.xpack.core.security.action.CreateApiKeyRequest;
+import org.elasticsearch.xpack.core.security.action.CreateApiKeyResponse;
+import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.security.authc.ApiKeyService;
+
+/**
+ * Implementation of the action needed to create an API key
+ */
+public final class TransportCreateApiKeyAction extends HandledTransportAction<CreateApiKeyRequest, CreateApiKeyResponse> {
+
+    private final ApiKeyService apiKeyService;
+    private final SecurityContext securityContext;
+
+    @Inject
+    public TransportCreateApiKeyAction(TransportService transportService, ActionFilters actionFilters, ApiKeyService apiKeyService,
+                                       SecurityContext context) {
+        super(CreateApiKeyAction.NAME, transportService, actionFilters, (Writeable.Reader<CreateApiKeyRequest>) CreateApiKeyRequest::new);
+        this.apiKeyService = apiKeyService;
+        this.securityContext = context;
+    }
+
+    @Override
+    protected void doExecute(Task task, CreateApiKeyRequest request, ActionListener<CreateApiKeyResponse> listener) {
+        final Authentication authentication = securityContext.getAuthentication();
+        if (authentication == null) {
+            listener.onFailure(new IllegalStateException("authentication is required"));
+        } else {
+            apiKeyService.createApiKey(authentication, request, listener);
+        }
+    }
+}

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportGetApiKeyAction.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security.action;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.security.action.GetApiKeyAction;
+import org.elasticsearch.xpack.core.security.action.GetApiKeyRequest;
+import org.elasticsearch.xpack.core.security.action.GetApiKeyResponse;
+import org.elasticsearch.xpack.security.authc.ApiKeyService;
+
+public final class TransportGetApiKeyAction extends HandledTransportAction<GetApiKeyRequest,GetApiKeyResponse> {
+
+    private final ApiKeyService apiKeyService;
+
+    @Inject
+    public TransportGetApiKeyAction(TransportService transportService, ActionFilters actionFilters, ApiKeyService apiKeyService) {
+        super(GetApiKeyAction.NAME, transportService, actionFilters,
+                (Writeable.Reader<GetApiKeyRequest>) GetApiKeyRequest::new);
+        this.apiKeyService = apiKeyService;
+    }
+
+    @Override
+    protected void doExecute(Task task, GetApiKeyRequest request, ActionListener<GetApiKeyResponse> listener) {
+        if (Strings.hasText(request.getRealmName()) || Strings.hasText(request.getUserName())) {
+            apiKeyService.getApiKeysForRealmAndUser(request.getRealmName(), request.getUserName(), listener);
+        } else if (Strings.hasText(request.getApiKeyId())) {
+            apiKeyService.getApiKeyForApiKeyId(request.getApiKeyId(), listener);
+        } else if (Strings.hasText(request.getApiKeyName())) {
+            apiKeyService.getApiKeyForApiKeyName(request.getApiKeyName(), listener);
+        } else {
+            listener.onFailure(new IllegalArgumentException("One of [api key id, api key name, username, realm name] must be specified"));
+        }
+    }
+
+}

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportInvalidateApiKeyAction.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security.action;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.security.action.InvalidateApiKeyAction;
+import org.elasticsearch.xpack.core.security.action.InvalidateApiKeyRequest;
+import org.elasticsearch.xpack.core.security.action.InvalidateApiKeyResponse;
+import org.elasticsearch.xpack.security.authc.ApiKeyService;
+
+public final class TransportInvalidateApiKeyAction extends HandledTransportAction<InvalidateApiKeyRequest, InvalidateApiKeyResponse> {
+
+    private final ApiKeyService apiKeyService;
+
+    @Inject
+    public TransportInvalidateApiKeyAction(TransportService transportService, ActionFilters actionFilters, ApiKeyService apiKeyService) {
+        super(InvalidateApiKeyAction.NAME, transportService, actionFilters,
+                (Writeable.Reader<InvalidateApiKeyRequest>) InvalidateApiKeyRequest::new);
+        this.apiKeyService = apiKeyService;
+    }
+
+    @Override
+    protected void doExecute(Task task, InvalidateApiKeyRequest request, ActionListener<InvalidateApiKeyResponse> listener) {
+        if (Strings.hasText(request.getRealmName()) || Strings.hasText(request.getUserName())) {
+            apiKeyService.invalidateApiKeysForRealmAndUser(request.getRealmName(), request.getUserName(), listener);
+        } else if (Strings.hasText(request.getId())) {
+            apiKeyService.invalidateApiKeyForApiKeyId(request.getId(), listener);
+        } else {
+            apiKeyService.invalidateApiKeyForApiKeyName(request.getName(), listener);
+        }
+    }
+
+}

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/interceptor/BulkShardRequestInterceptor.java

@@ -46,7 +46,7 @@ public class BulkShardRequestInterceptor implements RequestInterceptor<BulkShard
                     indicesAccessControl.getIndexPermissions(bulkItemRequest.index());
                 if (indexAccessControl != null) {
                     boolean fls = indexAccessControl.getFieldPermissions().hasFieldLevelSecurity();
-                    boolean dls = indexAccessControl.getQueries() != null;
+                    boolean dls = indexAccessControl.getDocumentPermissions().hasDocumentLevelPermissions();
                     if (fls || dls) {
                         if (bulkItemRequest.request() instanceof UpdateRequest) {
                             throw new ElasticsearchSecurityException("Can't execute a bulk request with update requests embedded if " +

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/interceptor/FieldAndDocumentLevelSecurityRequestInterceptor.java

@@ -40,7 +40,7 @@ abstract class FieldAndDocumentLevelSecurityRequestInterceptor<Request extends I
                 IndicesAccessControl.IndexAccessControl indexAccessControl = indicesAccessControl.getIndexPermissions(index);
                 if (indexAccessControl != null) {
                     boolean fieldLevelSecurityEnabled = indexAccessControl.getFieldPermissions().hasFieldLevelSecurity();
-                    boolean documentLevelSecurityEnabled = indexAccessControl.getQueries() != null;
+                    boolean documentLevelSecurityEnabled = indexAccessControl.getDocumentPermissions().hasDocumentLevelPermissions();
                     if (fieldLevelSecurityEnabled || documentLevelSecurityEnabled) {
                         if (fieldLevelSecurityEnabled || documentLevelSecurityEnabled) {
                             logger.trace("intercepted request for index [{}] with field level access controls [{}] document level access " +