Allow custom authorization with an authorization engine (#38358)

For some users, the built in authorization mechanism does not fit their
needs and no feature that we offer would allow them to control the
authorization process to meet their needs. In order to support this,
a concept of an AuthorizationEngine is being introduced, which can be
provided using the security extension mechanism.

An AuthorizationEngine is responsible for making the authorization
decisions about a request. The engine is responsible for knowing how to
authorize and can be backed by whatever mechanism a user wants. The
default mechanism is one backed by roles to provide the authorization
decisions. The AuthorizationEngine will be called by the
AuthorizationService, which handles more of the internal workings that
apply in general to authorization within Elasticsearch.

In order to support external authorization services that would back an
authorization engine, the entire authorization process has become
asynchronous, which also includes all calls to the AuthorizationEngine.

The use of roles also leaked out of the AuthorizationService in our
existing code that is not specifically related to roles so this also
needed to be addressed. RequestInterceptor instances sometimes used a
role to ensure a user was not attempting to escalate their privileges.
Addressing this leakage of roles meant that the RequestInterceptor
execution needed to move within the AuthorizationService and that
AuthorizationEngines needed to support detection of whether a user has
more privileges on a name than another. The second area where roles
leaked to the user is in the handling of a few privilege APIs that
could be used to retrieve the user's privileges or ask if a user has
privileges to perform an action. To remove the leakage of roles from
these actions, the AuthorizationService and AuthorizationEngine gained
methods that enabled an AuthorizationEngine to return the response for
these APIs.

Ultimately this feature is the work included in:
#37785
#37495
#37328
#36245
#38137
#38219

Closes #32435

build.gradle

@@ -233,6 +233,9 @@ allprojects {
     "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}": ':modules:aggs-matrix-stats',
     "org.elasticsearch.plugin:percolator-client:${version}": ':modules:percolator',
     "org.elasticsearch.plugin:rank-eval-client:${version}": ':modules:rank-eval',
+    // for security example plugins
+    "org.elasticsearch.plugin:x-pack-core:${version}": ':x-pack:plugin:core',
+    "org.elasticsearch.client.x-pack-transport:${version}": ':x-pack:transport-client'
   ]
 
   /*

plugins/examples/security-authorization-engine/build.gradle

@@ -0,0 +1,46 @@
+apply plugin: 'elasticsearch.esplugin'
+
+esplugin {
+  name 'security-authorization-engine'
+  description 'An example spi extension plugin for security that implements an Authorization Engine'
+  classname 'org.elasticsearch.example.AuthorizationEnginePlugin'
+  extendedPlugins = ['x-pack-security']
+}
+
+dependencies {
+  compileOnly "org.elasticsearch.plugin:x-pack-core:${version}"
+  testCompile "org.elasticsearch.client.x-pack-transport:${version}"
+}
+
+
+integTestRunner {
+    systemProperty 'tests.security.manager', 'false'
+}
+
+integTestCluster {
+  dependsOn buildZip
+  setting 'xpack.security.enabled', 'true'
+  setting 'xpack.ilm.enabled', 'false'
+  setting 'xpack.ml.enabled', 'false'
+  setting 'xpack.monitoring.enabled', 'false'
+  setting 'xpack.license.self_generated.type', 'trial'
+
+  // This is important, so that all the modules are available too.
+  // There are index templates that use token filters that are in analysis-module and
+  // processors are being used that are in ingest-common module.
+  distribution = 'default'
+
+  setupCommand 'setupDummyUser',
+               'bin/elasticsearch-users', 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'custom_superuser'
+  waitCondition = { node, ant ->
+    File tmpFile = new File(node.cwd, 'wait.success')
+    ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
+            dest: tmpFile.toString(),
+            username: 'test_user',
+            password: 'x-pack-test-password',
+            ignoreerrors: true,
+            retries: 10)
+    return tmpFile.exists()
+  }
+}
+check.dependsOn integTest

plugins/examples/security-authorization-engine/src/main/java/org/elasticsearch/example/AuthorizationEnginePlugin.java

@@ -0,0 +1,30 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.example;
+
+import org.elasticsearch.plugins.ActionPlugin;
+import org.elasticsearch.plugins.Plugin;
+
+/**
+ * Plugin class that is required so that the code contained here may be loaded as a plugin.
+ * Additional items such as settings and actions can be registered using this plugin class.
+ */
+public class AuthorizationEnginePlugin extends Plugin implements ActionPlugin {
+}

plugins/examples/security-authorization-engine/src/main/java/org/elasticsearch/example/CustomAuthorizationEngine.java

@@ -0,0 +1,238 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.example;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.cluster.metadata.AliasOrIndex;
+import org.elasticsearch.xpack.core.security.action.user.GetUserPrivilegesRequest;
+import org.elasticsearch.xpack.core.security.action.user.GetUserPrivilegesResponse;
+import org.elasticsearch.xpack.core.security.action.user.GetUserPrivilegesResponse.Indices;
+import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesRequest;
+import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesResponse;
+import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine;
+import org.elasticsearch.xpack.core.security.authz.ResolvedIndices;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.IndicesPrivileges;
+import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
+import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl.IndexAccessControl;
+import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissions;
+import org.elasticsearch.xpack.core.security.authz.permission.ResourcePrivileges;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
+import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivilege;
+import org.elasticsearch.xpack.core.security.user.User;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * A custom implementation of an authorization engine. This engine is extremely basic in that it
+ * authorizes based upon the name of a single role. If users have this role they are granted access.
+ */
+public class CustomAuthorizationEngine implements AuthorizationEngine {
+
+    @Override
+    public void resolveAuthorizationInfo(RequestInfo requestInfo, ActionListener<AuthorizationInfo> listener) {
+        final Authentication authentication = requestInfo.getAuthentication();
+        if (authentication.getUser().isRunAs()) {
+            final CustomAuthorizationInfo authenticatedUserAuthzInfo =
+                new CustomAuthorizationInfo(authentication.getUser().authenticatedUser().roles(), null);
+            listener.onResponse(new CustomAuthorizationInfo(authentication.getUser().roles(), authenticatedUserAuthzInfo));
+        } else {
+            listener.onResponse(new CustomAuthorizationInfo(authentication.getUser().roles(), null));
+        }
+    }
+
+    @Override
+    public void authorizeRunAs(RequestInfo requestInfo, AuthorizationInfo authorizationInfo, ActionListener<AuthorizationResult> listener) {
+        if (isSuperuser(requestInfo.getAuthentication().getUser().authenticatedUser())) {
+            listener.onResponse(AuthorizationResult.granted());
+        } else {
+            listener.onResponse(AuthorizationResult.deny());
+        }
+    }
+
+    @Override
+    public void authorizeClusterAction(RequestInfo requestInfo, AuthorizationInfo authorizationInfo,
+                                       ActionListener<AuthorizationResult> listener) {
+        if (isSuperuser(requestInfo.getAuthentication().getUser())) {
+            listener.onResponse(AuthorizationResult.granted());
+        } else {
+            listener.onResponse(AuthorizationResult.deny());
+        }
+    }
+
+    @Override
+    public void authorizeIndexAction(RequestInfo requestInfo, AuthorizationInfo authorizationInfo,
+                                     AsyncSupplier<ResolvedIndices> indicesAsyncSupplier,
+                                     Map<String, AliasOrIndex> aliasOrIndexLookup,
+                                     ActionListener<IndexAuthorizationResult> listener) {
+        if (isSuperuser(requestInfo.getAuthentication().getUser())) {
+            indicesAsyncSupplier.getAsync(ActionListener.wrap(resolvedIndices -> {
+                Map<String, IndexAccessControl> indexAccessControlMap = new HashMap<>();
+                for (String name : resolvedIndices.getLocal()) {
+                    indexAccessControlMap.put(name, new IndexAccessControl(true, FieldPermissions.DEFAULT, null));
+                }
+                IndicesAccessControl indicesAccessControl =
+                    new IndicesAccessControl(true, Collections.unmodifiableMap(indexAccessControlMap));
+                listener.onResponse(new IndexAuthorizationResult(true, indicesAccessControl));
+            }, listener::onFailure));
+        } else {
+            listener.onResponse(new IndexAuthorizationResult(true, IndicesAccessControl.DENIED));
+        }
+    }
+
+    @Override
+    public void loadAuthorizedIndices(RequestInfo requestInfo, AuthorizationInfo authorizationInfo,
+                                      Map<String, AliasOrIndex> aliasOrIndexLookup, ActionListener<List<String>> listener) {
+        if (isSuperuser(requestInfo.getAuthentication().getUser())) {
+            listener.onResponse(new ArrayList<>(aliasOrIndexLookup.keySet()));
+        } else {
+            listener.onResponse(Collections.emptyList());
+        }
+    }
+
+    @Override
+    public void validateIndexPermissionsAreSubset(RequestInfo requestInfo, AuthorizationInfo authorizationInfo,
+                                                  Map<String, List<String>> indexNameToNewNames,
+                                                  ActionListener<AuthorizationResult> listener) {
+        if (isSuperuser(requestInfo.getAuthentication().getUser())) {
+            listener.onResponse(AuthorizationResult.granted());
+        } else {
+            listener.onResponse(AuthorizationResult.deny());
+        }
+    }
+
+    @Override
+    public void checkPrivileges(Authentication authentication, AuthorizationInfo authorizationInfo,
+                                HasPrivilegesRequest hasPrivilegesRequest,
+                                Collection<ApplicationPrivilegeDescriptor> applicationPrivilegeDescriptors,
+                                ActionListener<HasPrivilegesResponse> listener) {
+        if (isSuperuser(authentication.getUser())) {
+            listener.onResponse(getHasPrivilegesResponse(authentication, hasPrivilegesRequest, true));
+        } else {
+            listener.onResponse(getHasPrivilegesResponse(authentication, hasPrivilegesRequest, false));
+        }
+    }
+
+    @Override
+    public void getUserPrivileges(Authentication authentication, AuthorizationInfo authorizationInfo, GetUserPrivilegesRequest request,
+                                  ActionListener<GetUserPrivilegesResponse> listener) {
+        if (isSuperuser(authentication.getUser())) {
+            listener.onResponse(getUserPrivilegesResponse(true));
+        } else {
+            listener.onResponse(getUserPrivilegesResponse(false));
+        }
+    }
+
+    private HasPrivilegesResponse getHasPrivilegesResponse(Authentication authentication, HasPrivilegesRequest hasPrivilegesRequest,
+                                                           boolean authorized) {
+        Map<String, Boolean> clusterPrivMap = new HashMap<>();
+        for (String clusterPriv : hasPrivilegesRequest.clusterPrivileges()) {
+            clusterPrivMap.put(clusterPriv, authorized);
+        }
+        final Map<String, ResourcePrivileges> indices = new LinkedHashMap<>();
+        for (IndicesPrivileges check : hasPrivilegesRequest.indexPrivileges()) {
+            for (String index : check.getIndices()) {
+                final Map<String, Boolean> privileges = new HashMap<>();
+                final ResourcePrivileges existing = indices.get(index);
+                if (existing != null) {
+                    privileges.putAll(existing.getPrivileges());
+                }
+                for (String privilege : check.getPrivileges()) {
+                    privileges.put(privilege, authorized);
+                }
+                indices.put(index, ResourcePrivileges.builder(index).addPrivileges(privileges).build());
+            }
+        }
+        final Map<String, Collection<ResourcePrivileges>> privilegesByApplication = new HashMap<>();
+        Set<String> applicationNames = Arrays.stream(hasPrivilegesRequest.applicationPrivileges())
+            .map(RoleDescriptor.ApplicationResourcePrivileges::getApplication)
+            .collect(Collectors.toSet());
+        for (String applicationName : applicationNames) {
+            final Map<String, ResourcePrivileges> appPrivilegesByResource = new LinkedHashMap<>();
+            for (RoleDescriptor.ApplicationResourcePrivileges p : hasPrivilegesRequest.applicationPrivileges()) {
+                if (applicationName.equals(p.getApplication())) {
+                    for (String resource : p.getResources()) {
+                        final Map<String, Boolean> privileges = new HashMap<>();
+                        final ResourcePrivileges existing = appPrivilegesByResource.get(resource);
+                        if (existing != null) {
+                            privileges.putAll(existing.getPrivileges());
+                        }
+                        for (String privilege : p.getPrivileges()) {
+                            privileges.put(privilege, authorized);
+                        }
+                        appPrivilegesByResource.put(resource, ResourcePrivileges.builder(resource).addPrivileges(privileges).build());
+                    }
+                }
+            }
+            privilegesByApplication.put(applicationName, appPrivilegesByResource.values());
+        }
+        return new HasPrivilegesResponse(authentication.getUser().principal(), authorized, clusterPrivMap, indices.values(),
+            privilegesByApplication);
+    }
+
+    private GetUserPrivilegesResponse getUserPrivilegesResponse(boolean isSuperuser) {
+        final Set<String> cluster = isSuperuser ? Collections.singleton("ALL") : Collections.emptySet();
+        final Set<ConditionalClusterPrivilege> conditionalCluster = Collections.emptySet();
+        final Set<GetUserPrivilegesResponse.Indices> indices = isSuperuser ? Collections.singleton(new Indices(Collections.singleton("*"),
+            Collections.singleton("*"), Collections.emptySet(), Collections.emptySet(), true)) : Collections.emptySet();
+
+        final Set<RoleDescriptor.ApplicationResourcePrivileges> application = isSuperuser ?
+            Collections.singleton(
+                RoleDescriptor.ApplicationResourcePrivileges.builder().application("*").privileges("*").resources("*").build()) :
+            Collections.emptySet();
+        final Set<String> runAs = isSuperuser ? Collections.singleton("*") : Collections.emptySet();
+        return new GetUserPrivilegesResponse(cluster, conditionalCluster, indices, application, runAs);
+    }
+
+    public static class CustomAuthorizationInfo implements AuthorizationInfo {
+
+        private final String[] roles;
+        private final CustomAuthorizationInfo authenticatedAuthzInfo;
+
+        CustomAuthorizationInfo(String[] roles, CustomAuthorizationInfo authenticatedAuthzInfo) {
+            this.roles = roles;
+            this.authenticatedAuthzInfo = authenticatedAuthzInfo;
+        }
+
+        @Override
+        public Map<String, Object> asMap() {
+            return Collections.singletonMap("roles", roles);
+        }
+
+        @Override
+        public CustomAuthorizationInfo getAuthenticatedUserAuthorizationInfo() {
+            return authenticatedAuthzInfo;
+        }
+    }
+
+    private boolean isSuperuser(User user) {
+        return Arrays.asList(user.roles()).contains("custom_superuser");
+    }
+}

plugins/examples/security-authorization-engine/src/main/java/org/elasticsearch/example/ExampleAuthorizationEngineExtension.java

@@ -0,0 +1,35 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.example;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.xpack.core.security.SecurityExtension;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine;
+
+/**
+ * Security extension class that registers the custom authorization engine to be used
+ */
+public class ExampleAuthorizationEngineExtension implements SecurityExtension {
+
+    @Override
+    public AuthorizationEngine getAuthorizationEngine(Settings settings) {
+        return new CustomAuthorizationEngine();
+    }
+}

plugins/examples/security-authorization-engine/src/main/resources/META-INF/services/org.elasticsearch.xpack.core.security.SecurityExtension

@@ -0,0 +1 @@
+org.elasticsearch.example.ExampleAuthorizationEngineExtension

plugins/examples/security-authorization-engine/src/test/java/org/elasticsearch/example/CustomAuthorizationEngineIT.java

@@ -0,0 +1,163 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.example;
+
+import org.apache.http.util.EntityUtils;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.common.network.NetworkModule;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.xpack.core.XPackClientPlugin;
+import org.elasticsearch.xpack.core.security.authc.support.Hasher;
+import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
+import org.elasticsearch.xpack.core.security.client.SecurityClient;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+
+import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.is;
+
+/**
+ * Integration tests for the custom authorization engine. These tests are meant to be run against
+ * an external cluster with the custom authorization plugin installed to validate the functionality
+ * when running as a plugin
+ */
+public class CustomAuthorizationEngineIT extends ESIntegTestCase {
+
+    @Override
+    protected Settings externalClusterClientSettings() {
+        final String token = "Basic " +
+            Base64.getEncoder().encodeToString(("test_user:x-pack-test-password").getBytes(StandardCharsets.UTF_8));
+        return Settings.builder()
+            .put(ThreadContext.PREFIX + ".Authorization", token)
+            .put(NetworkModule.TRANSPORT_TYPE_KEY, "security4")
+            .build();
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> transportClientPlugins() {
+        return Collections.singleton(XPackClientPlugin.class);
+    }
+
+    public void testClusterAction() throws IOException {
+        SecurityClient securityClient = new SecurityClient(client());
+        securityClient.preparePutUser("custom_user", "x-pack-test-password".toCharArray(), Hasher.BCRYPT, "custom_superuser").get();
+
+        {
+            RequestOptions.Builder options = RequestOptions.DEFAULT.toBuilder();
+            options.addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
+                basicAuthHeaderValue("custom_user", new SecureString("x-pack-test-password".toCharArray())));
+            Request request = new Request("GET", "_cluster/health");
+            request.setOptions(options);
+            Response response = getRestClient().performRequest(request);
+            assertThat(response.getStatusLine().getStatusCode(), is(200));
+        }
+
+        {
+            securityClient.preparePutUser("custom_user2", "x-pack-test-password".toCharArray(), Hasher.BCRYPT, "not_superuser").get();
+            RequestOptions.Builder options = RequestOptions.DEFAULT.toBuilder();
+            options.addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
+                basicAuthHeaderValue("custom_user2", new SecureString("x-pack-test-password".toCharArray())));
+            Request request = new Request("GET", "_cluster/health");
+            request.setOptions(options);
+            ResponseException e = expectThrows(ResponseException.class, () -> getRestClient().performRequest(request));
+            assertThat(e.getResponse().getStatusLine().getStatusCode(), is(403));
+        }
+    }
+
+    public void testIndexAction() throws IOException {
+        SecurityClient securityClient = new SecurityClient(client());
+        securityClient.preparePutUser("custom_user", "x-pack-test-password".toCharArray(), Hasher.BCRYPT, "custom_superuser").get();
+
+        {
+            RequestOptions.Builder options = RequestOptions.DEFAULT.toBuilder();
+            options.addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
+                basicAuthHeaderValue("custom_user", new SecureString("x-pack-test-password".toCharArray())));
+            Request request = new Request("PUT", "/index");
+            request.setOptions(options);
+            Response response = getRestClient().performRequest(request);
+            assertThat(response.getStatusLine().getStatusCode(), is(200));
+        }
+
+        {
+            securityClient.preparePutUser("custom_user2", "x-pack-test-password".toCharArray(), Hasher.BCRYPT, "not_superuser").get();
+            RequestOptions.Builder options = RequestOptions.DEFAULT.toBuilder();
+            options.addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
+                basicAuthHeaderValue("custom_user2", new SecureString("x-pack-test-password".toCharArray())));
+            Request request = new Request("PUT", "/index");
+            request.setOptions(options);
+            ResponseException e = expectThrows(ResponseException.class, () -> getRestClient().performRequest(request));
+            assertThat(e.getResponse().getStatusLine().getStatusCode(), is(403));
+        }
+    }
+
+    public void testRunAs() throws IOException {
+        SecurityClient securityClient = new SecurityClient(client());
+        securityClient.preparePutUser("custom_user", "x-pack-test-password".toCharArray(), Hasher.BCRYPT, "custom_superuser").get();
+        securityClient.preparePutUser("custom_user2", "x-pack-test-password".toCharArray(), Hasher.BCRYPT, "custom_superuser").get();
+        securityClient.preparePutUser("custom_user3", "x-pack-test-password".toCharArray(), Hasher.BCRYPT, "not_superuser").get();
+
+        {
+            RequestOptions.Builder options = RequestOptions.DEFAULT.toBuilder();
+            options.addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
+                basicAuthHeaderValue("custom_user", new SecureString("x-pack-test-password".toCharArray())));
+            options.addHeader("es-security-runas-user", "custom_user2");
+            Request request = new Request("GET", "/_security/_authenticate");
+            request.setOptions(options);
+            Response response = getRestClient().performRequest(request);
+            assertThat(response.getStatusLine().getStatusCode(), is(200));
+            String responseStr = EntityUtils.toString(response.getEntity());
+            assertThat(responseStr, containsString("custom_user2"));
+        }
+
+        {
+            RequestOptions.Builder options = RequestOptions.DEFAULT.toBuilder();
+            options.addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
+                basicAuthHeaderValue("custom_user", new SecureString("x-pack-test-password".toCharArray())));
+            options.addHeader("es-security-runas-user", "custom_user3");
+            Request request = new Request("PUT", "/index");
+            request.setOptions(options);
+            ResponseException e = expectThrows(ResponseException.class, () -> getRestClient().performRequest(request));
+            assertThat(e.getResponse().getStatusLine().getStatusCode(), is(403));
+        }
+
+        {
+            RequestOptions.Builder options = RequestOptions.DEFAULT.toBuilder();
+            options.addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
+                basicAuthHeaderValue("custom_user3", new SecureString("x-pack-test-password".toCharArray())));
+            options.addHeader("es-security-runas-user", "custom_user2");
+            Request request = new Request("PUT", "/index");
+            request.setOptions(options);
+            ResponseException e = expectThrows(ResponseException.class, () -> getRestClient().performRequest(request));
+            assertThat(e.getResponse().getStatusLine().getStatusCode(), is(403));
+        }
+    }
+}

plugins/examples/security-authorization-engine/src/test/java/org/elasticsearch/example/CustomAuthorizationEngineTests.java

@@ -0,0 +1,188 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.example;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.cluster.metadata.AliasOrIndex;
+import org.elasticsearch.cluster.metadata.AliasOrIndex.Index;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.transport.TransportRequest;
+import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationInfo;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationResult;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.IndexAuthorizationResult;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.RequestInfo;
+import org.elasticsearch.xpack.core.security.authz.ResolvedIndices;
+import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
+import org.elasticsearch.xpack.core.security.user.User;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.is;
+
+/**
+ * Unit tests for the custom authorization engine. These are basic tests that validate the
+ * engine's functionality outside of being used by the AuthorizationService
+ */
+public class CustomAuthorizationEngineTests extends ESTestCase {
+
+    public void testGetAuthorizationInfo() {
+        PlainActionFuture<AuthorizationInfo> future = new PlainActionFuture<>();
+        CustomAuthorizationEngine engine = new CustomAuthorizationEngine();
+        engine.resolveAuthorizationInfo(getRequestInfo(), future);
+        assertNotNull(future.actionGet());
+    }
+
+    public void testAuthorizeRunAs() {
+        final String action = "cluster:monitor/foo";
+        final TransportRequest request = new TransportRequest() {};
+        CustomAuthorizationEngine engine = new CustomAuthorizationEngine();
+        // unauthorized
+        {
+            Authentication authentication =
+                new Authentication(new User("joe", new String[]{"custom_superuser"}, new User("bar", "not_superuser")),
+                    new RealmRef("test", "test", "node"), new RealmRef("test", "test", "node"));
+            RequestInfo info = new RequestInfo(authentication, request, action);
+            PlainActionFuture<AuthorizationInfo> future = new PlainActionFuture<>();
+            engine.resolveAuthorizationInfo(info, future);
+            AuthorizationInfo authzInfo = future.actionGet();
+
+            PlainActionFuture<AuthorizationResult> resultFuture = new PlainActionFuture<>();
+            engine.authorizeRunAs(info, authzInfo, resultFuture);
+            AuthorizationResult result = resultFuture.actionGet();
+            assertThat(result.isGranted(), is(false));
+            assertThat(result.isAuditable(), is(true));
+        }
+
+        // authorized
+        {
+            Authentication authentication =
+                new Authentication(new User("joe", new String[]{"not_superuser"}, new User("bar", "custom_superuser")),
+                    new RealmRef("test", "test", "node"), new RealmRef("test", "test", "node"));
+            RequestInfo info = new RequestInfo(authentication, request, action);
+            PlainActionFuture<AuthorizationInfo> future = new PlainActionFuture<>();
+            engine.resolveAuthorizationInfo(info, future);
+            AuthorizationInfo authzInfo = future.actionGet();
+            PlainActionFuture<AuthorizationResult> resultFuture = new PlainActionFuture<>();
+            engine.authorizeRunAs(info, authzInfo, resultFuture);
+            AuthorizationResult result = resultFuture.actionGet();
+            assertThat(result.isGranted(), is(true));
+            assertThat(result.isAuditable(), is(true));
+        }
+    }
+
+    public void testAuthorizeClusterAction() {
+        CustomAuthorizationEngine engine = new CustomAuthorizationEngine();
+        RequestInfo requestInfo = getRequestInfo();
+        // authorized
+        {
+            PlainActionFuture<AuthorizationInfo> future = new PlainActionFuture<>();
+            engine.resolveAuthorizationInfo(requestInfo, future);
+            AuthorizationInfo authzInfo = future.actionGet();
+
+            PlainActionFuture<AuthorizationResult> resultFuture = new PlainActionFuture<>();
+            engine.authorizeClusterAction(requestInfo, authzInfo, resultFuture);
+            AuthorizationResult result = resultFuture.actionGet();
+            assertThat(result.isGranted(), is(true));
+            assertThat(result.isAuditable(), is(true));
+        }
+
+        // unauthorized
+        {
+            RequestInfo unauthReqInfo =
+                new RequestInfo(new Authentication(new User("joe", "not_superuser"), new RealmRef("test", "test", "node"), null),
+                    requestInfo.getRequest(), requestInfo.getAction());
+            PlainActionFuture<AuthorizationInfo> future = new PlainActionFuture<>();
+            engine.resolveAuthorizationInfo(unauthReqInfo, future);
+            AuthorizationInfo authzInfo = future.actionGet();
+
+            PlainActionFuture<AuthorizationResult> resultFuture = new PlainActionFuture<>();
+            engine.authorizeClusterAction(unauthReqInfo, authzInfo, resultFuture);
+            AuthorizationResult result = resultFuture.actionGet();
+            assertThat(result.isGranted(), is(false));
+            assertThat(result.isAuditable(), is(true));
+        }
+    }
+
+    public void testAuthorizeIndexAction() {
+        CustomAuthorizationEngine engine = new CustomAuthorizationEngine();
+        Map<String, AliasOrIndex> aliasOrIndexMap = new HashMap<>();
+        aliasOrIndexMap.put("index", new Index(IndexMetaData.builder("index")
+            .settings(Settings.builder().put("index.version.created", Version.CURRENT))
+            .numberOfShards(1)
+            .numberOfReplicas(0)
+            .build()));
+        // authorized
+        {
+            RequestInfo requestInfo =
+                new RequestInfo(new Authentication(new User("joe", "custom_superuser"), new RealmRef("test", "test", "node"), null),
+                    new SearchRequest(), "indices:data/read/search");
+            PlainActionFuture<AuthorizationInfo> future = new PlainActionFuture<>();
+            engine.resolveAuthorizationInfo(requestInfo, future);
+            AuthorizationInfo authzInfo = future.actionGet();
+
+            PlainActionFuture<IndexAuthorizationResult> resultFuture = new PlainActionFuture<>();
+            engine.authorizeIndexAction(requestInfo, authzInfo,
+                listener -> listener.onResponse(new ResolvedIndices(Collections.singletonList("index"), Collections.emptyList())),
+                aliasOrIndexMap, resultFuture);
+            IndexAuthorizationResult result = resultFuture.actionGet();
+            assertThat(result.isGranted(), is(true));
+            assertThat(result.isAuditable(), is(true));
+            IndicesAccessControl indicesAccessControl = result.getIndicesAccessControl();
+            assertNotNull(indicesAccessControl.getIndexPermissions("index"));
+            assertThat(indicesAccessControl.getIndexPermissions("index").isGranted(), is(true));
+        }
+
+        // unauthorized
+        {
+            RequestInfo requestInfo =
+                new RequestInfo(new Authentication(new User("joe", "not_superuser"), new RealmRef("test", "test", "node"), null),
+                    new SearchRequest(), "indices:data/read/search");
+            PlainActionFuture<AuthorizationInfo> future = new PlainActionFuture<>();
+            engine.resolveAuthorizationInfo(requestInfo, future);
+            AuthorizationInfo authzInfo = future.actionGet();
+
+            PlainActionFuture<IndexAuthorizationResult> resultFuture = new PlainActionFuture<>();
+            engine.authorizeIndexAction(requestInfo, authzInfo,
+                listener -> listener.onResponse(new ResolvedIndices(Collections.singletonList("index"), Collections.emptyList())),
+                aliasOrIndexMap, resultFuture);
+            IndexAuthorizationResult result = resultFuture.actionGet();
+            assertThat(result.isGranted(), is(false));
+            assertThat(result.isAuditable(), is(true));
+            IndicesAccessControl indicesAccessControl = result.getIndicesAccessControl();
+            assertNull(indicesAccessControl.getIndexPermissions("index"));
+        }
+    }
+
+    private RequestInfo getRequestInfo() {
+        final String action = "cluster:monitor/foo";
+        final TransportRequest request = new TransportRequest() {};
+        final Authentication authentication =
+            new Authentication(new User("joe", "custom_superuser"), new RealmRef("test", "test", "node"), null);
+        return new RequestInfo(authentication, request, action);
+    }
+}

x-pack/docs/build.gradle

@@ -13,7 +13,6 @@ buildRestTests.expectedUnconvertedCandidates = [
         'en/security/authentication/user-cache.asciidoc',
         'en/security/authorization/run-as-privilege.asciidoc',
         'en/security/ccs-clients-integrations/http.asciidoc',
-        'en/security/authorization/custom-roles-provider.asciidoc',
         'en/rest-api/watcher/stats.asciidoc',
         'en/watcher/example-watches/watching-time-series-data.asciidoc',
 ]

x-pack/docs/en/security/authorization/custom-roles-provider.asciidoc → x-pack/docs/en/security/authorization/custom-authorization.asciidoc

@@ -1,23 +1,23 @@
 [role="xpack"]
-[[custom-roles-provider]]
-=== Custom roles provider extension
+[[custom-roles-authorization]]
+=== Customizing roles and authorization
 
 If you need to retrieve user roles from a system not supported out-of-the-box
-by the {es} {security-features}, you can create a custom roles provider to
-retrieve and resolve
-roles. You implement a custom roles provider as an SPI loaded security extension
-as part of an ordinary elasticsearch plugin.
+or if the authorization system that is provided by the {es} {security-features}
+does not meet your needs, a SPI loaded security extension can be implemented to
+customize role retrieval and/or the authorization system. The SPI loaded
+security extension is part of an ordinary elasticsearch plugin.
 
 [[implementing-custom-roles-provider]]
 ==== Implementing a custom roles provider
 
-To create a custom roles provider: 
+To create a custom roles provider:
 
 . Implement the interface `BiConsumer<Set<String>, ActionListener<Set<RoleDescriptor>>>`.
   That is to say, the implementation consists of one method that takes a set of strings,
   which are the role names to resolve, and an ActionListener, on which the set of resolved
   role descriptors are passed on as the response.
-. The custom roles provider implementation must take special care to not block on any I/O 
+. The custom roles provider implementation must take special care to not block on any I/O
   operations.  It is the responsibility of the implementation to ensure asynchronous behavior
   and non-blocking calls, which is made easier by the fact that the `ActionListener` is
   provided on which to send the response when the roles have been resolved and the response
@@ -32,7 +32,7 @@ To package your custom roles provider as a plugin:
 [source,java]
 ----------------------------------------------------
 @Override
-public List<BiConsumer<Set<String>, ActionListener<Set<RoleDescriptor>>>> 
+public List<BiConsumer<Set<String>, ActionListener<Set<RoleDescriptor>>>>
 getRolesProviders(Settings settings, ResourceWatcherService resourceWatcherService) {
     ...
 }
@@ -41,50 +41,81 @@ getRolesProviders(Settings settings, ResourceWatcherService resourceWatcherServi
 The `getRolesProviders` method is used to provide a list of custom roles providers that
 will be used to resolve role names, if the role names could not be resolved by the reserved
 roles or native roles stores.   The list should be returned in the order that the custom role
-providers should be invoked to resolve roles.  For example, if `getRolesProviders` returns two 
-instances of roles providers, and both of them are able to resolve role `A`, then the resolved 
-role descriptor that will be used for role `A` will be the one resolved by the first roles 
+providers should be invoked to resolve roles.  For example, if `getRolesProviders` returns two
+instances of roles providers, and both of them are able to resolve role `A`, then the resolved
+role descriptor that will be used for role `A` will be the one resolved by the first roles
 provider in the list.
+
+[[implementing-authorization-engine]]
+==== Implementing an authorization engine
+
+To create an authorization engine, you need to:
+
+. Implement the `org.elasticsearch.xpack.core.security.authz.AuthorizationEngine`
+  interface in a class with the desired authorization behavior.
+. Implement the `org.elasticsearch.xpack.core.security.authz.Authorization.AuthorizationInfo`
+  interface in a class that contains the necessary information to authorize the request.
+
+To package your authorization engine as a plugin:
+
+. Implement an extension class for your authorization engine that extends
+  `org.elasticsearch.xpack.core.security.SecurityExtension`. There you need to
+  override the following method:
 +
 [source,java]
 ----------------------------------------------------
 @Override
-public List<String> getSettingsFilter() {
+public AuthorizationEngine getAuthorizationEngine(Settings settings) {
     ...
 }
 ----------------------------------------------------
 +
-The `Plugin#getSettingsFilter` method returns a list of setting names that should be
-filtered from the settings APIs as they may contain sensitive credentials. Note this method is not
-part of the `SecurityExtension` interface, it's available as part of the elasticsearch plugin main class.
+The `getAuthorizationEngine` method is used to provide the authorization engine
+implementation.
+
+Sample code that illustrates the structure and implementation of a custom
+authorization engine is provided in the
+https://github.com/elastic/elasticsearch/tree/master/plugin/examples/security-example-authorization-engine[elasticsearch]
+repository on GitHub. You can use this code as a starting point for creating your
+own authorization engine.
+
+[[packing-extension-plugin]]
+==== Implement an elasticsearch plugin
+
+In order to register the security extension for your custom roles provider or
+authorization engine, you need to also implement an elasticsearch plugin that
+contains the extension:
 
+. Implement a plugin class that extends `org.elasticsearch.plugins.Plugin`
 . Create a build configuration file for the plugin; Gradle is our recommendation.
+. Create a `plugin-descriptor.properties` file as described in
+  {plugins}/plugin-authors.html[Help for plugin authors].
 . Create a `META-INF/services/org.elasticsearch.xpack.core.security.SecurityExtension` descriptor file for the
   extension that contains the fully qualified class name of your `org.elasticsearch.xpack.core.security.SecurityExtension` implementation
 . Bundle all in a single zip file.
 
-[[using-custom-roles-provider]]
-==== Using a custom roles provider to resolve roles
+[[using-security-extension]]
+==== Using the security extension
 
-To use a custom roles provider:
+To use a security extension:
 
-. Install the roles provider extension on each node in the cluster. You run
+. Install the plugin with the extension on each node in the cluster. You run
   `bin/elasticsearch-plugin` with the `install` sub-command and specify the URL
   pointing to the zip file that contains the extension. For example:
 +
 [source,shell]
 ----------------------------------------
-bin/elasticsearch-plugin install file:///<path>/my-roles-provider-1.0.zip
+bin/elasticsearch-plugin install file:///<path>/my-extension-plugin-1.0.zip
 ----------------------------------------
 
-. Add any configuration parameters for any of the custom roles provider implementations
-to `elasticsearch.yml`.  The settings are not namespaced and you have access to any
-settings when constructing the custom roles providers, although it is recommended to
-have a namespacing convention for custom roles providers to keep your `elasticsearch.yml`
-configuration easy to understand.  
+. Add any configuration parameters for implementations in the extension to the
+`elasticsearch.yml` file.  The settings are not namespaced and you have access to any
+settings when constructing the extensions, although it is recommended to have a
+namespacing convention for extensions to keep your `elasticsearch.yml`
+configuration easy to understand.
 +
-For example, if you have a custom roles provider that 
-resolves roles from reading a blob in an S3 bucket on AWS, then you would specify settings 
+For example, if you have a custom roles provider that
+resolves roles from reading a blob in an S3 bucket on AWS, then you would specify settings
 in `elasticsearch.yml` such as:
 +
 [source,js]
@@ -94,8 +125,8 @@ custom_roles_provider.s3_roles_provider.region: us-east-1
 custom_roles_provider.s3_roles_provider.secret_key: xxx
 custom_roles_provider.s3_roles_provider.access_key: xxx
 ----------------------------------------
+// NOTCONSOLE
 +
-These settings will be available as the first parameter in the `getRolesProviders` method, from
-where you will create and return the custom roles provider instances.
+These settings are passed as arguments to the methods in the `SecurityExtension` interface.
 
 . Restart Elasticsearch.

x-pack/docs/en/security/authorization/managing-roles.asciidoc

@@ -179,7 +179,7 @@ There are two available mechanisms to define roles: using the _Role Management A
 or in local files on the {es} nodes. You can also implement
 custom roles providers.  If you need to integrate with another system to retrieve
 user roles, you can build a custom roles provider plugin. For more information,
-see <<custom-roles-provider, Custom Roles Provider Extension>>.
+see <<custom-roles-authorization, Customizing Roles and Authorization>>.
 
 [float]
 [[roles-management-ui]]

x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java

@@ -103,7 +103,8 @@ public class XPackLicenseState {
                             "The following X-Pack security functionality will be disabled: authentication, authorization, " +
                                 "ip filtering, and auditing. Please restart your node after applying the license.",
                             "Field and document level access control will be disabled.",
-                            "Custom realms will be ignored."
+                            "Custom realms will be ignored.",
+                            "A custom authorization engine will be ignored."
                         };
                 }
                 break;
@@ -116,7 +117,8 @@ public class XPackLicenseState {
                     case PLATINUM:
                         return new String[] {
                             "Field and document level access control will be disabled.",
-                            "Custom realms will be ignored."
+                            "Custom realms will be ignored.",
+                            "A custom authorization engine will be ignored."
                         };
                 }
                 break;
@@ -131,7 +133,8 @@ public class XPackLicenseState {
                             "Authentication will be limited to the native realms.",
                             "IP filtering and auditing will be disabled.",
                             "Field and document level access control will be disabled.",
-                            "Custom realms will be ignored."
+                            "Custom realms will be ignored.",
+                            "A custom authorization engine will be ignored."
                         };
                 }
         }
@@ -433,6 +436,17 @@ public class XPackLicenseState {
             && status.active;
     }
 
+    /**
+     * @return whether a custom authorization engine is allowed based on the license {@link OperationMode}
+     * @see org.elasticsearch.xpack.core.security.authc.support.DelegatedAuthorizationSettings
+     */
+    public synchronized boolean isAuthorizationEngineAllowed() {
+        final boolean isSecurityCurrentlyEnabled =
+            isSecurityEnabled(status.mode, isSecurityExplicitlyEnabled, isSecurityEnabled);
+        return isSecurityCurrentlyEnabled && (status.mode == OperationMode.PLATINUM || status.mode == OperationMode.TRIAL)
+            && status.active;
+    }
+
     /**
      * Determine if Watcher is available based on the current license.
      * <p>

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityExtension.java

@@ -11,6 +11,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.watcher.ResourceWatcherService;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationFailureHandler;
 import org.elasticsearch.xpack.core.security.authc.Realm;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.store.RoleRetrievalResult;
 
@@ -79,6 +80,18 @@ public interface SecurityExtension {
         return Collections.emptyList();
     }
 
+    /**
+     * Returns a authorization engine for authorizing requests, or null to use the default authorization mechanism.
+     *
+     * Only one installed extension may have an authorization engine. If more than
+     * one extension returns a non-null authorization engine, an error is raised.
+     *
+     * @param settings The configured settings for the node
+     */
+    default AuthorizationEngine getAuthorizationEngine(Settings settings) {
+        return null;
+    }
+
     /**
      * Loads the XPackSecurityExtensions from the given class loader
      */

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/AuthorizationEngine.java

@@ -0,0 +1,338 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.authz;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.cluster.metadata.AliasOrIndex;
+import org.elasticsearch.transport.TransportRequest;
+import org.elasticsearch.xpack.core.security.action.user.GetUserPrivilegesRequest;
+import org.elasticsearch.xpack.core.security.action.user.GetUserPrivilegesResponse;
+import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesRequest;
+import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesResponse;
+import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
+import org.elasticsearch.xpack.core.security.user.User;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * <p>
+ * An AuthorizationEngine is responsible for making the core decisions about whether a request
+ * should be authorized or not. The engine can and usually will be called multiple times during
+ * the authorization of a request. Security categorizes requests into a few different buckets
+ * and uses the action name as the indicator of what a request will perform. Internally, the
+ * action name is used to map a {@link TransportRequest} to the actual
+ * {@link org.elasticsearch.action.support.TransportAction} that will handle the request.
+ * </p><br>
+ * <p>
+ * Requests can be a <em>cluster</em> request or an <em>indices</em> request. Cluster requests
+ * are requests that tend to be global in nature; they could affect the whole cluster.
+ * Indices requests are those that deal with specific indices; the actions could have the affect
+ * of reading data, modifying data, creating an index, deleting an index, or modifying metadata.
+ * </p><br>
+ * <p>
+ * Each call to the engine will contain a {@link RequestInfo} object that contains the request,
+ * action name, and the authentication associated with the request. This data is provided by the
+ * engine so that all information about the request can be used to make the authorization decision.
+ * </p><br>
+ * The methods of the engine will be called in the following order:
+ * <ol>
+ *     <li>{@link #resolveAuthorizationInfo(RequestInfo, ActionListener)} to retrieve information
+ *         necessary to authorize the given user. It is important to note that the {@link RequestInfo}
+ *         may contain an {@link Authentication} object that actually has two users when the
+ *         <i>run as</i> feature is used and this method should resolve the information for both.
+ *         To check for the presence of run as, use the {@link User#isRunAs()} method on the user
+ *         retrieved using the {@link Authentication#getUser()} method.</li>
+ *     <li>{@link #authorizeRunAs(RequestInfo, AuthorizationInfo, ActionListener)} if the request
+ *         is making use of the run as feature. This method is used to ensure the authenticated user
+ *         can actually impersonate the user running the request.</li>
+ *     <li>{@link #authorizeClusterAction(RequestInfo, AuthorizationInfo, ActionListener)} if the
+ *         request is a cluster level operation.</li>
+ *     <li>{@link #authorizeIndexAction(RequestInfo, AuthorizationInfo, AsyncSupplier, Map, ActionListener)} if
+ *         the request is a an index action. This method may be called multiple times for a single
+ *         request as the request may be made up of sub-requests that also need to be authorized. The async supplier
+ *         for resolved indices will invoke the
+ *         {@link #loadAuthorizedIndices(RequestInfo, AuthorizationInfo, Map, ActionListener)} method
+ *         if it is used as part of the authorization process.</li>
+ * </ol>
+ * <br><p>
+ * <em>NOTE:</em> the {@link #loadAuthorizedIndices(RequestInfo, AuthorizationInfo, Map, ActionListener)}
+ * method may be called prior to {@link #authorizeIndexAction(RequestInfo, AuthorizationInfo, AsyncSupplier, Map, ActionListener)}
+ * in cases where wildcards need to be expanded.
+ * </p><br>
+ * Authorization engines can be called from various threads including network threads that should
+ * not be blocked waiting for I/O. Network threads in elasticsearch are limited and we rely on
+ * asynchronous processing to ensure optimal use of network threads; this is unlike many other Java
+ * based servers that have a thread for each concurrent request and blocking operations could take
+ * place on those threads. Given this it is imperative that the implementations used here do not
+ * block when calling out to an external service or waiting on some data.
+ */
+public interface AuthorizationEngine {
+
+    /**
+     * Asynchronously resolves any necessary information to authorize the given user(s). This could
+     * include retrieval of permissions from an index or external system.
+     *
+     * @param requestInfo object contain the request and associated information such as the action
+     *                    and associated user(s)
+     * @param listener the listener to be notified of success using {@link ActionListener#onResponse(Object)}
+     *                 or failure using {@link ActionListener#onFailure(Exception)}
+     */
+    void resolveAuthorizationInfo(RequestInfo requestInfo, ActionListener<AuthorizationInfo> listener);
+
+    /**
+     * Asynchronously authorizes an attempt for a user to run as another user.
+     *
+     * @param requestInfo object contain the request and associated information such as the action
+     *                    and associated user(s)
+     * @param authorizationInfo information needed from authorization that was previously retrieved
+     *                          from {@link #resolveAuthorizationInfo(RequestInfo, ActionListener)}
+     * @param listener the listener to be notified of the authorization result
+     */
+    void authorizeRunAs(RequestInfo requestInfo, AuthorizationInfo authorizationInfo, ActionListener<AuthorizationResult> listener);
+
+    /**
+     * Asynchronously authorizes a cluster action.
+     *
+     * @param requestInfo object contain the request and associated information such as the action
+     *                    and associated user(s)
+     * @param authorizationInfo information needed from authorization that was previously retrieved
+     *                          from {@link #resolveAuthorizationInfo(RequestInfo, ActionListener)}
+     * @param listener the listener to be notified of the authorization result
+     */
+    void authorizeClusterAction(RequestInfo requestInfo, AuthorizationInfo authorizationInfo, ActionListener<AuthorizationResult> listener);
+
+    /**
+     * Asynchronously authorizes an action that operates on an index. The indices and aliases that
+     * the request is attempting to operate on can be retrieved using the {@link AsyncSupplier} for
+     * {@link ResolvedIndices}. The resolved indices will contain the exact list of indices and aliases
+     * that the request is attempting to take action on; in other words this supplier handles wildcard
+     * expansion and datemath expressions.
+     *
+     * @param requestInfo object contain the request and associated information such as the action
+     *                    and associated user(s)
+     * @param authorizationInfo information needed from authorization that was previously retrieved
+     *                          from {@link #resolveAuthorizationInfo(RequestInfo, ActionListener)}
+     * @param indicesAsyncSupplier the asynchronous supplier for the indices that this request is
+     *                             attempting to operate on
+     * @param aliasOrIndexLookup a map of a string name to the cluster metadata specific to that
+     *                            alias or index
+     * @param listener the listener to be notified of the authorization result
+     */
+    void authorizeIndexAction(RequestInfo requestInfo, AuthorizationInfo authorizationInfo,
+                              AsyncSupplier<ResolvedIndices> indicesAsyncSupplier, Map<String, AliasOrIndex> aliasOrIndexLookup,
+                              ActionListener<IndexAuthorizationResult> listener);
+
+    /**
+     * Asynchronously loads a list of alias and index names for which the user is authorized
+     * to execute the requested action.
+     *
+     * @param requestInfo object contain the request and associated information such as the action
+     *                    and associated user(s)
+     * @param authorizationInfo information needed from authorization that was previously retrieved
+     *                          from {@link #resolveAuthorizationInfo(RequestInfo, ActionListener)}
+     * @param aliasOrIndexLookup a map of a string name to the cluster metadata specific to that
+     *                            alias or index
+     * @param listener the listener to be notified of the authorization result
+     */
+    void loadAuthorizedIndices(RequestInfo requestInfo, AuthorizationInfo authorizationInfo,
+                               Map<String, AliasOrIndex> aliasOrIndexLookup, ActionListener<List<String>> listener);
+
+
+    /**
+     * Asynchronously checks that the permissions a user would have for a given list of names do
+     * not exceed their permissions for a given name. This is used to ensure that a user cannot
+     * perform operations that would escalate their privileges over the data. Some examples include
+     * adding an alias to gain more permissions to a given index and/or resizing an index in order
+     * to gain more privileges on the data since the index name changes.
+     *
+     * @param requestInfo object contain the request and associated information such as the action
+     *                    and associated user(s)
+     * @param authorizationInfo information needed from authorization that was previously retrieved
+     *                          from {@link #resolveAuthorizationInfo(RequestInfo, ActionListener)}
+     * @param indexNameToNewNames A map of an existing index/alias name to a one or more names of
+     *                            an index/alias that the user is requesting to create. The method
+     *                            should validate that none of the names have more permissions than
+     *                            the name in the key would have.
+     * @param listener the listener to be notified of the authorization result
+     */
+    void validateIndexPermissionsAreSubset(RequestInfo requestInfo, AuthorizationInfo authorizationInfo,
+                                           Map<String, List<String>> indexNameToNewNames, ActionListener<AuthorizationResult> listener);
+
+    /**
+     * Checks the current user's privileges against those that being requested to check in the
+     * request. This provides a way for an application to ask if a user has permission to perform
+     * an action or if they have permissions to an application resource.
+     *
+     * @param authentication the authentication that is associated with this request
+     * @param authorizationInfo information needed from authorization that was previously retrieved
+     *                          from {@link #resolveAuthorizationInfo(RequestInfo, ActionListener)}
+     * @param hasPrivilegesRequest the request that contains the privileges to check for the user
+     * @param applicationPrivilegeDescriptors a collection of application privilege descriptors
+     * @param listener the listener to be notified of the has privileges response
+     */
+    void checkPrivileges(Authentication authentication, AuthorizationInfo authorizationInfo, HasPrivilegesRequest hasPrivilegesRequest,
+                         Collection<ApplicationPrivilegeDescriptor> applicationPrivilegeDescriptors,
+                         ActionListener<HasPrivilegesResponse> listener);
+
+    /**
+     * Retrieve's the current user's privileges in a standard format that can be rendered via an
+     * API for an application to understand the privileges that the current user has.
+     *
+     * @param authentication the authentication that is associated with this request
+     * @param authorizationInfo information needed from authorization that was previously retrieved
+     *                          from {@link #resolveAuthorizationInfo(RequestInfo, ActionListener)}
+     * @param request the request for retrieving the user's privileges
+     * @param listener the listener to be notified of the has privileges response
+     */
+    void getUserPrivileges(Authentication authentication, AuthorizationInfo authorizationInfo, GetUserPrivilegesRequest request,
+                           ActionListener<GetUserPrivilegesResponse> listener);
+
+    /**
+     * Interface for objects that contains the information needed to authorize a request
+     */
+    interface AuthorizationInfo {
+
+        /**
+         * @return a map representation of the authorization information. This map will be used to
+         * augment the data that is audited, so in the case of RBAC this map could contain the
+         * role names.
+         */
+        Map<String, Object> asMap();
+
+        /**
+         * This method should be overridden in case of run as. Authorization info is only retrieved
+         * a single time and should represent the information to authorize both run as and the
+         * operation being performed.
+         */
+        default AuthorizationInfo getAuthenticatedUserAuthorizationInfo() {
+            return this;
+        }
+    }
+
+    /**
+     * Implementation of authorization info that is used in cases where we were not able to resolve
+     * the authorization info
+     */
+    final class EmptyAuthorizationInfo implements AuthorizationInfo {
+
+        public static final EmptyAuthorizationInfo INSTANCE = new EmptyAuthorizationInfo();
+
+        private EmptyAuthorizationInfo() {}
+
+        @Override
+        public Map<String, Object> asMap() {
+            return Collections.emptyMap();
+        }
+    }
+
+    /**
+     * A class that encapsulates information about the request that is being authorized including
+     * the actual transport request, the authentication, and the action being invoked.
+     */
+    final class RequestInfo {
+
+        private final Authentication authentication;
+        private final TransportRequest request;
+        private final String action;
+
+        public RequestInfo(Authentication authentication, TransportRequest request, String action) {
+            this.authentication = authentication;
+            this.request = request;
+            this.action = action;
+        }
+
+        public String getAction() {
+            return action;
+        }
+
+        public Authentication getAuthentication() {
+            return authentication;
+        }
+
+        public TransportRequest getRequest() {
+            return request;
+        }
+    }
+
+    /**
+     * Represents the result of authorization. This includes whether the actions should be granted
+     * and if this should be considered an auditable event.
+     */
+    class AuthorizationResult {
+
+        private final boolean granted;
+        private final boolean auditable;
+
+        /**
+         * Create an authorization result with the provided granted value that is auditable
+         */
+        public AuthorizationResult(boolean granted) {
+            this(granted, true);
+        }
+
+        public AuthorizationResult(boolean granted, boolean auditable) {
+            this.granted = granted;
+            this.auditable = auditable;
+        }
+
+        public boolean isGranted() {
+            return granted;
+        }
+
+        public boolean isAuditable() {
+            return auditable;
+        }
+
+        /**
+         * Returns a new authorization result that is granted and auditable
+         */
+        public static AuthorizationResult granted() {
+            return new AuthorizationResult(true);
+        }
+
+        /**
+         * Returns a new authorization result that is denied and auditable
+         */
+        public static AuthorizationResult deny() {
+            return new AuthorizationResult(false);
+        }
+    }
+
+    /**
+     * An extension of {@link AuthorizationResult} that is specific to index requests. Index requests
+     * need to return a {@link IndicesAccessControl} object representing the users permissions to indices
+     * that are being operated on.
+     */
+    class IndexAuthorizationResult extends AuthorizationResult {
+
+        private final IndicesAccessControl indicesAccessControl;
+
+        public IndexAuthorizationResult(boolean auditable, IndicesAccessControl indicesAccessControl) {
+            super(indicesAccessControl == null || indicesAccessControl.isGranted(), auditable);
+            this.indicesAccessControl = indicesAccessControl;
+        }
+
+        public IndicesAccessControl getIndicesAccessControl() {
+            return indicesAccessControl;
+        }
+    }
+
+    @FunctionalInterface
+    interface AsyncSupplier<V> {
+
+        /**
+         * Asynchronously retrieves the value that is being supplied and notifies the listener upon
+         * completion.
+         */
+        void getAsync(ActionListener<V> listener);
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/ResolvedIndices.java

@@ -0,0 +1,111 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.authz;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import static org.elasticsearch.xpack.core.security.authz.IndicesAndAliasesResolverField.NO_INDEX_PLACEHOLDER;
+
+/**
+ * Stores a collection of index names separated into "local" and "remote".
+ * This allows the resolution and categorization to take place exactly once per-request.
+ */
+public final class ResolvedIndices {
+    private final List<String> local;
+    private final List<String> remote;
+
+    public ResolvedIndices(List<String> local, List<String> remote) {
+        this.local = Collections.unmodifiableList(local);
+        this.remote = Collections.unmodifiableList(remote);
+    }
+
+    /**
+     * Returns the collection of index names that have been stored as "local" indices.
+     * This is a <code>List</code> because order may be important. For example <code>[ "a*" , "-a1" ]</code> is interpreted differently
+     * to <code>[ "-a1", "a*" ]</code>. As a consequence, this list <em>may contain duplicates</em>.
+     */
+    public List<String> getLocal() {
+        return local;
+    }
+
+    /**
+     * Returns the collection of index names that have been stored as "remote" indices.
+     */
+    public List<String> getRemote() {
+        return remote;
+    }
+
+    /**
+     * @return <code>true</code> if both the {@link #getLocal() local} and {@link #getRemote() remote} index lists are empty.
+     */
+    public boolean isEmpty() {
+        return local.isEmpty() && remote.isEmpty();
+    }
+
+    /**
+     * @return <code>true</code> if the {@link #getRemote() remote} index lists is empty, and the local index list contains the
+     * {@link IndicesAndAliasesResolverField#NO_INDEX_PLACEHOLDER no-index-placeholder} and nothing else.
+     */
+    public boolean isNoIndicesPlaceholder() {
+        return remote.isEmpty() && local.size() == 1 && local.contains(NO_INDEX_PLACEHOLDER);
+    }
+
+    public String[] toArray() {
+        final String[] array = new String[local.size() + remote.size()];
+        int i = 0;
+        for (String index : local) {
+            array[i++] = index;
+        }
+        for (String index : remote) {
+            array[i++] = index;
+        }
+        return array;
+    }
+
+    /**
+     * Builder class for ResolvedIndices that allows for the building of a list of indices
+     * without the need to construct new objects and merging them together
+     */
+    public static class Builder {
+
+        private final List<String> local = new ArrayList<>();
+        private final List<String> remote = new ArrayList<>();
+
+        /** add a local index name */
+        public void addLocal(String index) {
+            local.add(index);
+        }
+
+        /** adds the array of local index names */
+        public void addLocal(String[] indices) {
+            local.addAll(Arrays.asList(indices));
+        }
+
+        /** adds the list of local index names */
+        public void addLocal(List<String> indices) {
+            local.addAll(indices);
+        }
+
+        /** adds the list of remote index names */
+        public void addRemote(List<String> indices) {
+            remote.addAll(indices);
+        }
+
+        /** @return <code>true</code> if both the local and remote index lists are empty. */
+        public boolean isEmpty() {
+            return local.isEmpty() && remote.isEmpty();
+        }
+
+        /** @return a immutable ResolvedIndices instance with the local and remote index lists */
+        public ResolvedIndices build() {
+            return new ResolvedIndices(local, remote);
+        }
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/IndicesAccessControl.java

@@ -25,6 +25,7 @@ public class IndicesAccessControl {
     public static final IndicesAccessControl ALLOW_NO_INDICES = new IndicesAccessControl(true,
             Collections.singletonMap(IndicesAndAliasesResolverField.NO_INDEX_PLACEHOLDER,
                     new IndicesAccessControl.IndexAccessControl(true, new FieldPermissions(), DocumentPermissions.allowAll())));
+    public static final IndicesAccessControl DENIED = new IndicesAccessControl(false, Collections.emptyMap());
 
     private final boolean granted;
     private final Map<String, IndexAccessControl> indexPermissions;

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java

@@ -12,7 +12,6 @@ import org.apache.lucene.util.automaton.TooComplexToDeterminizeException;
 import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.cluster.metadata.AliasOrIndex;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -31,7 +30,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
-import java.util.SortedMap;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.function.Predicate;
@@ -182,11 +180,10 @@ public final class IndicesPermission {
      * Authorizes the provided action against the provided indices, given the current cluster metadata
      */
     public Map<String, IndicesAccessControl.IndexAccessControl> authorize(String action, Set<String> requestedIndicesOrAliases,
-                                                                          MetaData metaData, FieldPermissionsCache fieldPermissionsCache) {
+                                                                          Map<String, AliasOrIndex> allAliasesAndIndices,
+                                                                          FieldPermissionsCache fieldPermissionsCache) {
         // now... every index that is associated with the request, must be granted
         // by at least one indices permission group
-
-        SortedMap<String, AliasOrIndex> allAliasesAndIndices = metaData.getAliasAndIndexLookup();
         Map<String, Set<FieldPermissions>> fieldPermissionsByIndex = new HashMap<>();
         Map<String, DocumentLevelPermissions> roleQueriesByIndex = new HashMap<>();
         Map<String, Boolean> grantedBuilder = new HashMap<>();

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/LimitedRole.java

@@ -6,20 +6,23 @@
 
 package org.elasticsearch.xpack.core.security.authz.permission;
 
-import org.elasticsearch.cluster.metadata.MetaData;
+import org.apache.lucene.util.automaton.Automaton;
+import org.elasticsearch.cluster.metadata.AliasOrIndex;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
 import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
 import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilege;
+import org.elasticsearch.xpack.core.security.support.Automatons;
 
 import java.util.Collection;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.function.Predicate;
 
 /**
  * A {@link Role} limited by another role.<br>
- * The effective permissions returned on {@link #authorize(String, Set, MetaData, FieldPermissionsCache)} call would be limited by the
+ * The effective permissions returned on {@link #authorize(String, Set, Map, FieldPermissionsCache)} call would be limited by the
  * provided role.
  */
 public final class LimitedRole extends Role {
@@ -37,10 +40,32 @@ public final class LimitedRole extends Role {
     }
 
     @Override
-    public IndicesAccessControl authorize(String action, Set<String> requestedIndicesOrAliases, MetaData metaData,
+    public ClusterPermission cluster() {
+        throw new UnsupportedOperationException("cannot retrieve cluster permission on limited role");
+    }
+
+    @Override
+    public IndicesPermission indices() {
+        throw new UnsupportedOperationException("cannot retrieve indices permission on limited role");
+    }
+
+    @Override
+    public ApplicationPermission application() {
+        throw new UnsupportedOperationException("cannot retrieve application permission on limited role");
+    }
+
+    @Override
+    public RunAsPermission runAs() {
+        throw new UnsupportedOperationException("cannot retrieve cluster permission on limited role");
+    }
+
+    @Override
+    public IndicesAccessControl authorize(String action, Set<String> requestedIndicesOrAliases,
+                                          Map<String, AliasOrIndex> aliasAndIndexLookup,
                                           FieldPermissionsCache fieldPermissionsCache) {
-        IndicesAccessControl indicesAccessControl = super.authorize(action, requestedIndicesOrAliases, metaData, fieldPermissionsCache);
-        IndicesAccessControl limitedByIndicesAccessControl = limitedBy.authorize(action, requestedIndicesOrAliases, metaData,
+        IndicesAccessControl indicesAccessControl =
+            super.authorize(action, requestedIndicesOrAliases, aliasAndIndexLookup, fieldPermissionsCache);
+        IndicesAccessControl limitedByIndicesAccessControl = limitedBy.authorize(action, requestedIndicesOrAliases, aliasAndIndexLookup,
                 fieldPermissionsCache);
 
         return indicesAccessControl.limitIndicesAccessControl(limitedByIndicesAccessControl);
@@ -52,11 +77,18 @@ public final class LimitedRole extends Role {
      */
     @Override
     public Predicate<String> allowedIndicesMatcher(String action) {
-        Predicate<String> predicate = indices().allowedIndicesMatcher(action);
+        Predicate<String> predicate = super.indices().allowedIndicesMatcher(action);
         predicate = predicate.and(limitedBy.indices().allowedIndicesMatcher(action));
         return predicate;
     }
 
+    @Override
+    public Automaton allowedActionsMatcher(String index) {
+        final Automaton allowedMatcher = super.allowedActionsMatcher(index);
+        final Automaton limitedByMatcher = super.allowedActionsMatcher(index);
+        return Automatons.intersectAndMinimize(allowedMatcher, limitedByMatcher);
+    }
+
     /**
      * Check if indices permissions allow for the given action, also checks whether the limited by role allows the given actions
      *
@@ -137,6 +169,11 @@ public final class LimitedRole extends Role {
         return ResourcePrivilegesMap.intersection(resourcePrivilegesMap, resourcePrivilegesMapForLimitedRole);
     }
 
+    @Override
+    public boolean checkRunAs(String runAs) {
+        return super.checkRunAs(runAs) && limitedBy.checkRunAs(runAs);
+    }
+
     /**
      * Create a new role defined by given role and the limited role.
      *

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/Role.java

@@ -5,7 +5,8 @@
  */
 package org.elasticsearch.xpack.core.security.authz.permission;
 
-import org.elasticsearch.cluster.metadata.MetaData;
+import org.apache.lucene.util.automaton.Automaton;
+import org.elasticsearch.cluster.metadata.AliasOrIndex;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.Tuple;
@@ -82,7 +83,15 @@ public class Role {
      * has the privilege for executing the given action on.
      */
     public Predicate<String> allowedIndicesMatcher(String action) {
-        return indices().allowedIndicesMatcher(action);
+        return indices.allowedIndicesMatcher(action);
+    }
+
+    public Automaton allowedActionsMatcher(String index) {
+        return indices.allowedActionsMatcher(index);
+    }
+
+    public boolean checkRunAs(String runAsName) {
+        return runAs.check(runAsName);
     }
 
     /**
@@ -92,7 +101,7 @@ public class Role {
      * @return {@code true} if action is allowed else returns {@code false}
      */
     public boolean checkIndicesAction(String action) {
-        return indices().check(action);
+        return indices.check(action);
     }
 
 
@@ -108,7 +117,7 @@ public class Role {
      */
     public ResourcePrivilegesMap checkIndicesPrivileges(Set<String> checkForIndexPatterns, boolean allowRestrictedIndices,
                                                                  Set<String> checkForPrivileges) {
-        return indices().checkResourcePrivileges(checkForIndexPatterns, allowRestrictedIndices, checkForPrivileges);
+        return indices.checkResourcePrivileges(checkForIndexPatterns, allowRestrictedIndices, checkForPrivileges);
     }
 
     /**
@@ -119,7 +128,7 @@ public class Role {
      * @return {@code true} if action is allowed else returns {@code false}
      */
     public boolean checkClusterAction(String action, TransportRequest request) {
-        return cluster().check(action, request);
+        return cluster.check(action, request);
     }
 
     /**
@@ -129,7 +138,7 @@ public class Role {
      * @return {@code true} if cluster privilege is allowed else returns {@code false}
      */
     public boolean grants(ClusterPrivilege clusterPrivilege) {
-        return cluster().grants(clusterPrivilege);
+        return cluster.grants(clusterPrivilege);
     }
 
     /**
@@ -147,7 +156,7 @@ public class Role {
     public ResourcePrivilegesMap checkApplicationResourcePrivileges(final String applicationName, Set<String> checkForResources,
                                                                     Set<String> checkForPrivilegeNames,
                                                                     Collection<ApplicationPrivilegeDescriptor> storedPrivileges) {
-        return application().checkResourcePrivileges(applicationName, checkForResources, checkForPrivilegeNames, storedPrivileges);
+        return application.checkResourcePrivileges(applicationName, checkForResources, checkForPrivilegeNames, storedPrivileges);
     }
 
     /**
@@ -155,10 +164,11 @@ public class Role {
      * specified action with the requested indices/aliases. At the same time if field and/or document level security
      * is configured for any group also the allowed fields and role queries are resolved.
      */
-    public IndicesAccessControl authorize(String action, Set<String> requestedIndicesOrAliases, MetaData metaData,
+    public IndicesAccessControl authorize(String action, Set<String> requestedIndicesOrAliases,
+                                          Map<String, AliasOrIndex> aliasAndIndexLookup,
                                           FieldPermissionsCache fieldPermissionsCache) {
         Map<String, IndicesAccessControl.IndexAccessControl> indexPermissions = indices.authorize(
-            action, requestedIndicesOrAliases, metaData, fieldPermissionsCache
+            action, requestedIndicesOrAliases, aliasAndIndexLookup, fieldPermissionsCache
         );
 
         // At least one role / indices permission set need to match with all the requested indices/aliases:

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Exceptions.java

@@ -27,6 +27,10 @@ public class Exceptions {
     }
 
     public static ElasticsearchSecurityException authorizationError(String msg, Object... args) {
-        return new ElasticsearchSecurityException(msg, RestStatus.FORBIDDEN, args);
+        return new ElasticsearchSecurityException(msg, RestStatus.FORBIDDEN, null, args);
+    }
+
+    public static ElasticsearchSecurityException authorizationError(String msg, Exception cause, Object... args) {
+        return new ElasticsearchSecurityException(msg, RestStatus.FORBIDDEN, cause, args);
     }
 }

x-pack/plugin/core/src/test/java/org/elasticsearch/license/XPackLicenseStateTests.java

@@ -226,17 +226,17 @@ public class XPackLicenseStateTests extends ESTestCase {
     }
 
     public void testSecurityAckTrialStandardGoldOrPlatinumToBasic() {
-        assertAckMesssages(XPackField.SECURITY, randomTrialStandardGoldOrPlatinumMode(), BASIC, 3);
+        assertAckMesssages(XPackField.SECURITY, randomTrialStandardGoldOrPlatinumMode(), BASIC, 4);
     }
 
     public void testSecurityAckAnyToStandard() {
         OperationMode from = randomFrom(BASIC, GOLD, PLATINUM, TRIAL);
-        assertAckMesssages(XPackField.SECURITY, from, STANDARD, 4);
+        assertAckMesssages(XPackField.SECURITY, from, STANDARD, 5);
     }
 
     public void testSecurityAckBasicStandardTrialOrPlatinumToGold() {
         OperationMode from = randomFrom(BASIC, PLATINUM, TRIAL, STANDARD);
-        assertAckMesssages(XPackField.SECURITY, from, GOLD, 2);
+        assertAckMesssages(XPackField.SECURITY, from, GOLD, 3);
     }
 
     public void testMonitoringAckBasicToAny() {

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/permission/LimitedRoleTests.java

@@ -69,12 +69,14 @@ public class LimitedRoleTests extends ESTestCase {
         Role fromRole = Role.builder("a-role").cluster(Collections.singleton(ClusterPrivilegeName.MANAGE_SECURITY), Collections.emptyList())
                 .add(IndexPrivilege.ALL, "_index").add(IndexPrivilege.CREATE_INDEX, "_index1").build();
 
-        IndicesAccessControl iac = fromRole.authorize(SearchAction.NAME, Sets.newHashSet("_index", "_alias1"), md, fieldPermissionsCache);
+        IndicesAccessControl iac = fromRole.authorize(SearchAction.NAME, Sets.newHashSet("_index", "_alias1"), md.getAliasAndIndexLookup(),
+            fieldPermissionsCache);
         assertThat(iac.getIndexPermissions("_index"), is(notNullValue()));
         assertThat(iac.getIndexPermissions("_index").isGranted(), is(true));
         assertThat(iac.getIndexPermissions("_index1"), is(notNullValue()));
         assertThat(iac.getIndexPermissions("_index1").isGranted(), is(false));
-        iac = fromRole.authorize(CreateIndexAction.NAME, Sets.newHashSet("_index", "_index1"), md, fieldPermissionsCache);
+        iac = fromRole.authorize(CreateIndexAction.NAME, Sets.newHashSet("_index", "_index1"), md.getAliasAndIndexLookup(),
+            fieldPermissionsCache);
         assertThat(iac.getIndexPermissions("_index"), is(notNullValue()));
         assertThat(iac.getIndexPermissions("_index").isGranted(), is(true));
         assertThat(iac.getIndexPermissions("_index1"), is(notNullValue()));
@@ -84,34 +86,40 @@ public class LimitedRoleTests extends ESTestCase {
             Role limitedByRole = Role.builder("limited-role")
                     .cluster(Collections.singleton(ClusterPrivilegeName.ALL), Collections.emptyList()).add(IndexPrivilege.READ, "_index")
                     .add(IndexPrivilege.NONE, "_index1").build();
-            iac = limitedByRole.authorize(SearchAction.NAME, Sets.newHashSet("_index", "_alias1"), md, fieldPermissionsCache);
+            iac = limitedByRole.authorize(SearchAction.NAME, Sets.newHashSet("_index", "_alias1"), md.getAliasAndIndexLookup(),
+                fieldPermissionsCache);
             assertThat(iac.getIndexPermissions("_index"), is(notNullValue()));
             assertThat(iac.getIndexPermissions("_index").isGranted(), is(true));
             assertThat(iac.getIndexPermissions("_index1"), is(notNullValue()));
             assertThat(iac.getIndexPermissions("_index1").isGranted(), is(false));
-            iac = limitedByRole.authorize(DeleteIndexAction.NAME, Sets.newHashSet("_index", "_alias1"), md, fieldPermissionsCache);
+            iac = limitedByRole.authorize(DeleteIndexAction.NAME, Sets.newHashSet("_index", "_alias1"), md.getAliasAndIndexLookup(),
+                fieldPermissionsCache);
             assertThat(iac.getIndexPermissions("_index"), is(notNullValue()));
             assertThat(iac.getIndexPermissions("_index").isGranted(), is(false));
             assertThat(iac.getIndexPermissions("_index1"), is(notNullValue()));
             assertThat(iac.getIndexPermissions("_index1").isGranted(), is(false));
-            iac = limitedByRole.authorize(CreateIndexAction.NAME, Sets.newHashSet("_index", "_alias1"), md, fieldPermissionsCache);
+            iac = limitedByRole.authorize(CreateIndexAction.NAME, Sets.newHashSet("_index", "_alias1"), md.getAliasAndIndexLookup(),
+                fieldPermissionsCache);
             assertThat(iac.getIndexPermissions("_index"), is(notNullValue()));
             assertThat(iac.getIndexPermissions("_index").isGranted(), is(false));
             assertThat(iac.getIndexPermissions("_index1"), is(notNullValue()));
             assertThat(iac.getIndexPermissions("_index1").isGranted(), is(false));
 
             Role role = LimitedRole.createLimitedRole(fromRole, limitedByRole);
-            iac = role.authorize(SearchAction.NAME, Sets.newHashSet("_index", "_alias1"), md, fieldPermissionsCache);
+            iac = role.authorize(SearchAction.NAME, Sets.newHashSet("_index", "_alias1"), md.getAliasAndIndexLookup(),
+                fieldPermissionsCache);
             assertThat(iac.getIndexPermissions("_index"), is(notNullValue()));
             assertThat(iac.getIndexPermissions("_index").isGranted(), is(true));
             assertThat(iac.getIndexPermissions("_index1"), is(notNullValue()));
             assertThat(iac.getIndexPermissions("_index1").isGranted(), is(false));
-            iac = role.authorize(DeleteIndexAction.NAME, Sets.newHashSet("_index", "_alias1"), md, fieldPermissionsCache);
+            iac = role.authorize(DeleteIndexAction.NAME, Sets.newHashSet("_index", "_alias1"), md.getAliasAndIndexLookup(),
+                fieldPermissionsCache);
             assertThat(iac.getIndexPermissions("_index"), is(notNullValue()));
             assertThat(iac.getIndexPermissions("_index").isGranted(), is(false));
             assertThat(iac.getIndexPermissions("_index1"), is(notNullValue()));
             assertThat(iac.getIndexPermissions("_index1").isGranted(), is(false));
-            iac = role.authorize(CreateIndexAction.NAME, Sets.newHashSet("_index", "_index1"), md, fieldPermissionsCache);
+            iac = role.authorize(CreateIndexAction.NAME, Sets.newHashSet("_index", "_index1"), md.getAliasAndIndexLookup(),
+                fieldPermissionsCache);
             assertThat(iac.getIndexPermissions("_index"), is(notNullValue()));
             assertThat(iac.getIndexPermissions("_index").isGranted(), is(false));
             assertThat(iac.getIndexPermissions("_index1"), is(notNullValue()));

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java

@@ -41,6 +41,7 @@ import org.elasticsearch.action.search.MultiSearchAction;
 import org.elasticsearch.action.search.SearchAction;
 import org.elasticsearch.action.update.UpdateAction;
 import org.elasticsearch.cluster.metadata.AliasMetaData;
+import org.elasticsearch.cluster.metadata.AliasOrIndex;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.settings.Settings;
@@ -147,6 +148,7 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.SortedMap;
 
 import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.is;
@@ -586,8 +588,8 @@ public class ReservedRolesStoreTests extends ESTestCase {
                 GetSettingsAction.NAME, IndicesShardStoresAction.NAME, UpgradeStatusAction.NAME, RecoveryAction.NAME);
         for (final String indexMonitoringActionName : indexMonitoringActionNamesList) {
             final Map<String, IndexAccessControl> authzMap = role.indices().authorize(indexMonitoringActionName,
-                    Sets.newHashSet(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX, RestrictedIndicesNames.SECURITY_INDEX_NAME), metaData,
-                    fieldPermissionsCache);
+                Sets.newHashSet(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX, RestrictedIndicesNames.SECURITY_INDEX_NAME),
+                metaData.getAliasAndIndexLookup(), fieldPermissionsCache);
             assertThat(authzMap.get(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX).isGranted(), is(true));
             assertThat(authzMap.get(RestrictedIndicesNames.SECURITY_INDEX_NAME).isGranted(), is(true));
         }
@@ -704,22 +706,24 @@ public class ReservedRolesStoreTests extends ESTestCase {
                 .build();
 
         FieldPermissionsCache fieldPermissionsCache = new FieldPermissionsCache(Settings.EMPTY);
+        SortedMap<String, AliasOrIndex> lookup = metaData.getAliasAndIndexLookup();
         Map<String, IndexAccessControl> authzMap =
-                superuserRole.indices().authorize(SearchAction.NAME, Sets.newHashSet("a1", "ba"), metaData, fieldPermissionsCache);
+                superuserRole.indices().authorize(SearchAction.NAME, Sets.newHashSet("a1", "ba"), lookup, fieldPermissionsCache);
         assertThat(authzMap.get("a1").isGranted(), is(true));
         assertThat(authzMap.get("b").isGranted(), is(true));
-        authzMap = superuserRole.indices().authorize(DeleteIndexAction.NAME, Sets.newHashSet("a1", "ba"), metaData, fieldPermissionsCache);
+        authzMap =
+            superuserRole.indices().authorize(DeleteIndexAction.NAME, Sets.newHashSet("a1", "ba"), lookup, fieldPermissionsCache);
         assertThat(authzMap.get("a1").isGranted(), is(true));
         assertThat(authzMap.get("b").isGranted(), is(true));
-        authzMap = superuserRole.indices().authorize(IndexAction.NAME, Sets.newHashSet("a2", "ba"), metaData, fieldPermissionsCache);
+        authzMap = superuserRole.indices().authorize(IndexAction.NAME, Sets.newHashSet("a2", "ba"), lookup, fieldPermissionsCache);
         assertThat(authzMap.get("a2").isGranted(), is(true));
         assertThat(authzMap.get("b").isGranted(), is(true));
         authzMap = superuserRole.indices()
-                .authorize(UpdateSettingsAction.NAME, Sets.newHashSet("aaaaaa", "ba"), metaData, fieldPermissionsCache);
+                .authorize(UpdateSettingsAction.NAME, Sets.newHashSet("aaaaaa", "ba"), lookup, fieldPermissionsCache);
         assertThat(authzMap.get("aaaaaa").isGranted(), is(true));
         assertThat(authzMap.get("b").isGranted(), is(true));
         authzMap = superuserRole.indices().authorize(randomFrom(IndexAction.NAME, DeleteIndexAction.NAME, SearchAction.NAME),
-                Sets.newHashSet(RestrictedIndicesNames.SECURITY_INDEX_NAME), metaData, fieldPermissionsCache);
+                Sets.newHashSet(RestrictedIndicesNames.SECURITY_INDEX_NAME), lookup, fieldPermissionsCache);
         assertThat(authzMap.get(RestrictedIndicesNames.SECURITY_INDEX_NAME).isGranted(), is(true));
         assertThat(authzMap.get(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX).isGranted(), is(true));
         assertTrue(superuserRole.indices().check(SearchAction.NAME));

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java

@@ -114,6 +114,7 @@ import org.elasticsearch.xpack.core.security.authc.Realm;
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.RealmSettings;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine;
 import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField;
 import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
 import org.elasticsearch.xpack.core.security.authz.accesscontrol.SecurityIndexSearcherWrapper;
@@ -134,12 +135,6 @@ import org.elasticsearch.xpack.security.action.TransportCreateApiKeyAction;
 import org.elasticsearch.xpack.security.action.TransportGetApiKeyAction;
 import org.elasticsearch.xpack.security.action.TransportInvalidateApiKeyAction;
 import org.elasticsearch.xpack.security.action.filter.SecurityActionFilter;
-import org.elasticsearch.xpack.security.action.interceptor.BulkShardRequestInterceptor;
-import org.elasticsearch.xpack.security.action.interceptor.IndicesAliasesRequestInterceptor;
-import org.elasticsearch.xpack.security.action.interceptor.RequestInterceptor;
-import org.elasticsearch.xpack.security.action.interceptor.ResizeRequestInterceptor;
-import org.elasticsearch.xpack.security.action.interceptor.SearchRequestInterceptor;
-import org.elasticsearch.xpack.security.action.interceptor.UpdateRequestInterceptor;
 import org.elasticsearch.xpack.security.action.privilege.TransportDeletePrivilegesAction;
 import org.elasticsearch.xpack.security.action.privilege.TransportGetPrivilegesAction;
 import org.elasticsearch.xpack.security.action.privilege.TransportPutPrivilegesAction;
@@ -180,6 +175,12 @@ import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingSt
 import org.elasticsearch.xpack.security.authz.AuthorizationService;
 import org.elasticsearch.xpack.security.authz.SecuritySearchOperationListener;
 import org.elasticsearch.xpack.security.authz.accesscontrol.OptOutQueryCache;
+import org.elasticsearch.xpack.security.authz.interceptor.BulkShardRequestInterceptor;
+import org.elasticsearch.xpack.security.authz.interceptor.IndicesAliasesRequestInterceptor;
+import org.elasticsearch.xpack.security.authz.interceptor.RequestInterceptor;
+import org.elasticsearch.xpack.security.authz.interceptor.ResizeRequestInterceptor;
+import org.elasticsearch.xpack.security.authz.interceptor.SearchRequestInterceptor;
+import org.elasticsearch.xpack.security.authz.interceptor.UpdateRequestInterceptor;
 import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore;
 import org.elasticsearch.xpack.security.authz.store.FileRolesStore;
 import org.elasticsearch.xpack.security.authz.store.NativePrivilegeStore;
@@ -436,36 +437,22 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
         for (SecurityExtension extension : securityExtensions) {
             rolesProviders.addAll(extension.getRolesProviders(settings, resourceWatcherService));
         }
-        final CompositeRolesStore allRolesStore = new CompositeRolesStore(settings, fileRolesStore, nativeRolesStore,
-            reservedRolesStore, privilegeStore, rolesProviders, threadPool.getThreadContext(), getLicenseState(), fieldPermissionsCache);
+
+        final ApiKeyService apiKeyService = new ApiKeyService(settings, Clock.systemUTC(), client, securityIndex.get(), clusterService);
+        components.add(apiKeyService);
+        final CompositeRolesStore allRolesStore = new CompositeRolesStore(settings, fileRolesStore, nativeRolesStore, reservedRolesStore,
+            privilegeStore, rolesProviders, threadPool.getThreadContext(), getLicenseState(), fieldPermissionsCache, apiKeyService);
         securityIndex.get().addIndexStateListener(allRolesStore::onSecurityIndexStateChange);
         // to keep things simple, just invalidate all cached entries on license change. this happens so rarely that the impact should be
         // minimal
         getLicenseState().addListener(allRolesStore::invalidateAll);
 
-        final ApiKeyService apiKeyService = new ApiKeyService(settings, Clock.systemUTC(), client, securityIndex.get(), clusterService,
-                allRolesStore);
-        components.add(apiKeyService);
-
         final AuthenticationFailureHandler failureHandler = createAuthenticationFailureHandler(realms);
         authcService.set(new AuthenticationService(settings, realms, auditTrailService, failureHandler, threadPool,
                 anonymousUser, tokenService, apiKeyService));
         components.add(authcService.get());
         securityIndex.get().addIndexStateListener(authcService.get()::onSecurityIndexStateChange);
 
-        final AuthorizationService authzService = new AuthorizationService(settings, allRolesStore, clusterService,
-            auditTrailService, failureHandler, threadPool, anonymousUser, apiKeyService, fieldPermissionsCache);
-        components.add(nativeRolesStore); // used by roles actions
-        components.add(reservedRolesStore); // used by roles actions
-        components.add(allRolesStore); // for SecurityFeatureSet and clear roles cache
-        components.add(authzService);
-
-        ipFilter.set(new IPFilter(settings, auditTrailService, clusterService.getClusterSettings(), getLicenseState()));
-        components.add(ipFilter.get());
-        DestructiveOperations destructiveOperations = new DestructiveOperations(settings, clusterService.getClusterSettings());
-        securityInterceptor.set(new SecurityServerTransportInterceptor(settings, threadPool, authcService.get(),
-                authzService, getLicenseState(), getSslService(), securityContext.get(), destructiveOperations, clusterService));
-
         Set<RequestInterceptor> requestInterceptors = Sets.newHashSet(
             new ResizeRequestInterceptor(threadPool, getLicenseState(), auditTrailService),
             new IndicesAliasesRequestInterceptor(threadPool.getThreadContext(), getLicenseState(), auditTrailService));
@@ -478,12 +465,46 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
         }
         requestInterceptors = Collections.unmodifiableSet(requestInterceptors);
 
+        final AuthorizationService authzService = new AuthorizationService(settings, allRolesStore, clusterService,
+            auditTrailService, failureHandler, threadPool, anonymousUser, getAuthorizationEngine(), requestInterceptors,
+            getLicenseState());
+
+        components.add(nativeRolesStore); // used by roles actions
+        components.add(reservedRolesStore); // used by roles actions
+        components.add(allRolesStore); // for SecurityFeatureSet and clear roles cache
+        components.add(authzService);
+
+        ipFilter.set(new IPFilter(settings, auditTrailService, clusterService.getClusterSettings(), getLicenseState()));
+        components.add(ipFilter.get());
+        DestructiveOperations destructiveOperations = new DestructiveOperations(settings, clusterService.getClusterSettings());
+        securityInterceptor.set(new SecurityServerTransportInterceptor(settings, threadPool, authcService.get(),
+                authzService, getLicenseState(), getSslService(), securityContext.get(), destructiveOperations, clusterService));
+
         securityActionFilter.set(new SecurityActionFilter(authcService.get(), authzService, getLicenseState(),
-                requestInterceptors, threadPool, securityContext.get(), destructiveOperations));
+            threadPool, securityContext.get(), destructiveOperations));
 
         return components;
     }
 
+    private AuthorizationEngine getAuthorizationEngine() {
+        AuthorizationEngine authorizationEngine = null;
+        String extensionName = null;
+        for (SecurityExtension extension : securityExtensions) {
+            final AuthorizationEngine extensionEngine = extension.getAuthorizationEngine(settings);
+            if (extensionEngine != null && authorizationEngine != null) {
+                throw new IllegalStateException("Extensions [" + extensionName + "] and [" + extension.toString() + "] "
+                    + "both set an authorization engine");
+            }
+            authorizationEngine = extensionEngine;
+            extensionName = extension.toString();
+        }
+
+        if (authorizationEngine != null) {
+            logger.debug("Using authorization engine from extension [" + extensionName + "]");
+        }
+        return authorizationEngine;
+    }
+
     private AuthenticationFailureHandler createAuthenticationFailureHandler(final Realms realms) {
         AuthenticationFailureHandler failureHandler = null;
         String extensionName = null;

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportCreateApiKeyAction.java

@@ -19,6 +19,10 @@ import org.elasticsearch.xpack.core.security.action.CreateApiKeyRequest;
 import org.elasticsearch.xpack.core.security.action.CreateApiKeyResponse;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.security.authc.ApiKeyService;
+import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore;
+
+import java.util.Arrays;
+import java.util.HashSet;
 
 /**
  * Implementation of the action needed to create an API key
@@ -27,13 +31,15 @@ public final class TransportCreateApiKeyAction extends HandledTransportAction<Cr
 
     private final ApiKeyService apiKeyService;
     private final SecurityContext securityContext;
+    private final CompositeRolesStore rolesStore;
 
     @Inject
     public TransportCreateApiKeyAction(TransportService transportService, ActionFilters actionFilters, ApiKeyService apiKeyService,
-                                       SecurityContext context) {
+                                       SecurityContext context, CompositeRolesStore rolesStore) {
         super(CreateApiKeyAction.NAME, transportService, actionFilters, (Writeable.Reader<CreateApiKeyRequest>) CreateApiKeyRequest::new);
         this.apiKeyService = apiKeyService;
         this.securityContext = context;
+        this.rolesStore = rolesStore;
     }
 
     @Override
@@ -42,7 +48,9 @@ public final class TransportCreateApiKeyAction extends HandledTransportAction<Cr
         if (authentication == null) {
             listener.onFailure(new IllegalStateException("authentication is required"));
         } else {
-            apiKeyService.createApiKey(authentication, request, listener);
+            rolesStore.getRoleDescriptors(new HashSet<>(Arrays.asList(authentication.getUser().roles())),
+                ActionListener.wrap(roleDescriptors -> apiKeyService.createApiKey(authentication, request, roleDescriptors, listener),
+                    listener::onFailure));
         }
     }
 }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/filter/SecurityActionFilter.java

@@ -32,13 +32,11 @@ import org.elasticsearch.xpack.core.security.authz.privilege.HealthAndStatsPrivi
 import org.elasticsearch.xpack.core.security.support.Automatons;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
 import org.elasticsearch.xpack.security.action.SecurityActionMapper;
-import org.elasticsearch.xpack.security.action.interceptor.RequestInterceptor;
 import org.elasticsearch.xpack.security.authc.AuthenticationService;
 import org.elasticsearch.xpack.security.authz.AuthorizationService;
 import org.elasticsearch.xpack.security.authz.AuthorizationUtils;
 
 import java.io.IOException;
-import java.util.Set;
 import java.util.function.Predicate;
 
 public class SecurityActionFilter implements ActionFilter {
@@ -50,19 +48,17 @@ public class SecurityActionFilter implements ActionFilter {
     private final AuthenticationService authcService;
     private final AuthorizationService authzService;
     private final SecurityActionMapper actionMapper = new SecurityActionMapper();
-    private final Set<RequestInterceptor> requestInterceptors;
     private final XPackLicenseState licenseState;
     private final ThreadContext threadContext;
     private final SecurityContext securityContext;
     private final DestructiveOperations destructiveOperations;
 
     public SecurityActionFilter(AuthenticationService authcService, AuthorizationService authzService,
-                                XPackLicenseState licenseState, Set<RequestInterceptor> requestInterceptors, ThreadPool threadPool,
+                                XPackLicenseState licenseState, ThreadPool threadPool,
                                 SecurityContext securityContext, DestructiveOperations destructiveOperations) {
         this.authcService = authcService;
         this.authzService = authzService;
         this.licenseState = licenseState;
-        this.requestInterceptors = requestInterceptors;
         this.threadContext = threadPool.getThreadContext();
         this.securityContext = securityContext;
         this.destructiveOperations = destructiveOperations;
@@ -164,21 +160,8 @@ public class SecurityActionFilter implements ActionFilter {
         if (authentication == null) {
             listener.onFailure(new IllegalArgumentException("authentication must be non null for authorization"));
         } else {
-            final AuthorizationUtils.AsyncAuthorizer asyncAuthorizer = new AuthorizationUtils.AsyncAuthorizer(authentication, listener,
-                    (userRoles, runAsRoles) -> {
-                        authzService.authorize(authentication, securityAction, request, userRoles, runAsRoles);
-                        /*
-                         * We use a separate concept for code that needs to be run after authentication and authorization that could
-                         * affect the running of the action. This is done to make it more clear of the state of the request.
-                         */
-                        for (RequestInterceptor interceptor : requestInterceptors) {
-                            if (interceptor.supports(request)) {
-                                interceptor.intercept(request, authentication, runAsRoles != null ? runAsRoles : userRoles, securityAction);
-                            }
-                        }
-                        listener.onResponse(null);
-                    });
-            asyncAuthorizer.authorize(authzService);
+            authzService.authorize(authentication, securityAction, request, ActionListener.wrap(ignore -> listener.onResponse(null),
+                listener::onFailure));
         }
     }
 }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/interceptor/FieldAndDocumentLevelSecurityRequestInterceptor.java

@@ -1,61 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.security.action.interceptor;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.action.IndicesRequest;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.license.XPackLicenseState;
-import org.elasticsearch.xpack.core.security.authc.Authentication;
-import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField;
-import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
-import org.elasticsearch.xpack.core.security.authz.permission.Role;
-
-/**
- * Base class for interceptors that disables features when field level security is configured for indices a request
- * is going to execute on.
- */
-abstract class FieldAndDocumentLevelSecurityRequestInterceptor<Request extends IndicesRequest> implements
-        RequestInterceptor<Request> {
-
-    private final ThreadContext threadContext;
-    private final XPackLicenseState licenseState;
-    private final Logger logger;
-
-    FieldAndDocumentLevelSecurityRequestInterceptor(ThreadContext threadContext, XPackLicenseState licenseState) {
-        this.threadContext = threadContext;
-        this.licenseState = licenseState;
-        this.logger = LogManager.getLogger(getClass());
-    }
-
-    @Override
-    public void intercept(Request request, Authentication authentication, Role userPermissions, String action) {
-        if (licenseState.isDocumentAndFieldLevelSecurityAllowed()) {
-            final IndicesAccessControl indicesAccessControl = threadContext.getTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY);
-            for (String index : request.indices()) {
-                IndicesAccessControl.IndexAccessControl indexAccessControl = indicesAccessControl.getIndexPermissions(index);
-                if (indexAccessControl != null) {
-                    boolean fieldLevelSecurityEnabled = indexAccessControl.getFieldPermissions().hasFieldLevelSecurity();
-                    boolean documentLevelSecurityEnabled = indexAccessControl.getDocumentPermissions().hasDocumentLevelPermissions();
-                    if (fieldLevelSecurityEnabled || documentLevelSecurityEnabled) {
-                        if (fieldLevelSecurityEnabled || documentLevelSecurityEnabled) {
-                            logger.trace("intercepted request for index [{}] with field level access controls [{}] document level access " +
-                                    "controls [{}]. disabling conflicting features", index, fieldLevelSecurityEnabled,
-                                documentLevelSecurityEnabled);
-                        }
-                        disableFeatures(request, fieldLevelSecurityEnabled, documentLevelSecurityEnabled);
-                        return;
-                    }
-                }
-                logger.trace("intercepted request for index [{}] without field or document level access controls", index);
-            }
-        }
-    }
-
-    protected abstract void disableFeatures(Request request, boolean fieldLevelSecurityEnabled, boolean documentLevelSecurityEnabled);
-
-}

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/interceptor/IndicesAliasesRequestInterceptor.java

@@ -1,90 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.security.action.interceptor;
-
-import org.apache.lucene.util.automaton.Automaton;
-import org.apache.lucene.util.automaton.Operations;
-import org.elasticsearch.ElasticsearchSecurityException;
-import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.license.XPackLicenseState;
-import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.transport.TransportRequest;
-import org.elasticsearch.xpack.core.security.authc.Authentication;
-import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField;
-import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
-import org.elasticsearch.xpack.core.security.authz.permission.Role;
-import org.elasticsearch.xpack.core.security.support.Exceptions;
-import org.elasticsearch.xpack.security.audit.AuditTrailService;
-import org.elasticsearch.xpack.security.audit.AuditUtil;
-
-import java.util.HashMap;
-import java.util.Map;
-
-public final class IndicesAliasesRequestInterceptor implements RequestInterceptor<IndicesAliasesRequest> {
-
-    private final ThreadContext threadContext;
-    private final XPackLicenseState licenseState;
-    private final AuditTrailService auditTrailService;
-
-    public IndicesAliasesRequestInterceptor(ThreadContext threadContext, XPackLicenseState licenseState,
-                                            AuditTrailService auditTrailService) {
-        this.threadContext = threadContext;
-        this.licenseState = licenseState;
-        this.auditTrailService = auditTrailService;
-    }
-
-    @Override
-    public void intercept(IndicesAliasesRequest request, Authentication authentication, Role userPermissions, String action) {
-        final XPackLicenseState frozenLicenseState = licenseState.copyCurrentLicenseState();
-        if (frozenLicenseState.isAuthAllowed()) {
-            if (frozenLicenseState.isDocumentAndFieldLevelSecurityAllowed()) {
-                IndicesAccessControl indicesAccessControl = threadContext.getTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY);
-                for (IndicesAliasesRequest.AliasActions aliasAction : request.getAliasActions()) {
-                    if (aliasAction.actionType() == IndicesAliasesRequest.AliasActions.Type.ADD) {
-                        for (String index : aliasAction.indices()) {
-                            IndicesAccessControl.IndexAccessControl indexAccessControl = indicesAccessControl.getIndexPermissions(index);
-                            if (indexAccessControl != null) {
-                                final boolean fls = indexAccessControl.getFieldPermissions().hasFieldLevelSecurity();
-                                final boolean dls = indexAccessControl.getDocumentPermissions().hasDocumentLevelPermissions();
-                                if (fls || dls) {
-                                    throw new ElasticsearchSecurityException("Alias requests are not allowed for users who have " +
-                                        "field or document level security enabled on one of the indices", RestStatus.BAD_REQUEST);
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-
-            Map<String, Automaton> permissionsMap = new HashMap<>();
-            for (IndicesAliasesRequest.AliasActions aliasAction : request.getAliasActions()) {
-                if (aliasAction.actionType() == IndicesAliasesRequest.AliasActions.Type.ADD) {
-                    for (String index : aliasAction.indices()) {
-                        Automaton indexPermissions =
-                            permissionsMap.computeIfAbsent(index, userPermissions.indices()::allowedActionsMatcher);
-                        for (String alias : aliasAction.aliases()) {
-                            Automaton aliasPermissions =
-                                permissionsMap.computeIfAbsent(alias, userPermissions.indices()::allowedActionsMatcher);
-                            if (Operations.subsetOf(aliasPermissions, indexPermissions) == false) {
-                                // TODO we've already audited a access granted event so this is going to look ugly
-                                auditTrailService.accessDenied(AuditUtil.extractRequestId(threadContext), authentication, action, request,
-                                    userPermissions.names());
-                                throw Exceptions.authorizationError("Adding an alias is not allowed when the alias " +
-                                    "has more permissions than any of the indices");
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
-
-    @Override
-    public boolean supports(TransportRequest request) {
-        return request instanceof IndicesAliasesRequest;
-    }
-}

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/interceptor/RequestInterceptor.java

@@ -1,28 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.security.action.interceptor;
-
-import org.elasticsearch.transport.TransportRequest;
-import org.elasticsearch.xpack.core.security.authc.Authentication;
-import org.elasticsearch.xpack.core.security.authz.permission.Role;
-
-/**
- * A request interceptor can introspect a request and modify it.
- */
-public interface RequestInterceptor<Request> {
-
-    /**
-     * If {@link #supports(TransportRequest)} returns <code>true</code> this interceptor will introspect the request
-     * and potentially modify it.
-     */
-    void intercept(Request request, Authentication authentication, Role userPermissions, String action);
-
-    /**
-     * Returns whether this request interceptor should intercept the specified request.
-     */
-    boolean supports(TransportRequest request);
-
-}

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/interceptor/ResizeRequestInterceptor.java

@@ -1,75 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.security.action.interceptor;
-
-import org.apache.lucene.util.automaton.Automaton;
-import org.apache.lucene.util.automaton.Operations;
-import org.elasticsearch.ElasticsearchSecurityException;
-import org.elasticsearch.action.admin.indices.shrink.ResizeRequest;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.license.XPackLicenseState;
-import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.TransportRequest;
-import org.elasticsearch.xpack.core.security.authc.Authentication;
-import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField;
-import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
-import org.elasticsearch.xpack.core.security.authz.permission.Role;
-import org.elasticsearch.xpack.core.security.support.Exceptions;
-import org.elasticsearch.xpack.security.audit.AuditTrailService;
-
-import static org.elasticsearch.xpack.security.audit.AuditUtil.extractRequestId;
-
-public final class ResizeRequestInterceptor implements RequestInterceptor<ResizeRequest> {
-
-    private final ThreadContext threadContext;
-    private final XPackLicenseState licenseState;
-    private final AuditTrailService auditTrailService;
-
-    public ResizeRequestInterceptor(ThreadPool threadPool, XPackLicenseState licenseState,
-                                    AuditTrailService auditTrailService) {
-        this.threadContext = threadPool.getThreadContext();
-        this.licenseState = licenseState;
-        this.auditTrailService = auditTrailService;
-    }
-
-    @Override
-    public void intercept(ResizeRequest request, Authentication authentication, Role userPermissions, String action) {
-        final XPackLicenseState frozenLicenseState = licenseState.copyCurrentLicenseState();
-        if (frozenLicenseState.isAuthAllowed()) {
-            if (frozenLicenseState.isDocumentAndFieldLevelSecurityAllowed()) {
-                IndicesAccessControl indicesAccessControl =
-                    threadContext.getTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY);
-                IndicesAccessControl.IndexAccessControl indexAccessControl =
-                    indicesAccessControl.getIndexPermissions(request.getSourceIndex());
-                if (indexAccessControl != null) {
-                    final boolean fls = indexAccessControl.getFieldPermissions().hasFieldLevelSecurity();
-                    final boolean dls = indexAccessControl.getDocumentPermissions().hasDocumentLevelPermissions();
-                    if (fls || dls) {
-                        throw new ElasticsearchSecurityException("Resize requests are not allowed for users when " +
-                            "field or document level security is enabled on the source index", RestStatus.BAD_REQUEST);
-                    }
-                }
-            }
-
-            // ensure that the user would have the same level of access OR less on the target index
-            final Automaton sourceIndexPermissions = userPermissions.indices().allowedActionsMatcher(request.getSourceIndex());
-            final Automaton targetIndexPermissions =
-                userPermissions.indices().allowedActionsMatcher(request.getTargetIndexRequest().index());
-            if (Operations.subsetOf(targetIndexPermissions, sourceIndexPermissions) == false) {
-                // TODO we've already audited a access granted event so this is going to look ugly
-                auditTrailService.accessDenied(extractRequestId(threadContext), authentication, action, request, userPermissions.names());
-                throw Exceptions.authorizationError("Resizing an index is not allowed when the target index " +
-                    "has more permissions than the source index");
-            }
-        }
-    }
-
-    @Override
-    public boolean supports(TransportRequest request) {
-        return request instanceof ResizeRequest;
-    }
-}

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportGetUserPrivilegesAction.java

@@ -5,13 +5,9 @@
  */
 package org.elasticsearch.xpack.security.action.user;
 
-import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.apache.lucene.util.automaton.Operations;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -20,26 +16,9 @@ import org.elasticsearch.xpack.core.security.action.user.GetUserPrivilegesAction
 import org.elasticsearch.xpack.core.security.action.user.GetUserPrivilegesRequest;
 import org.elasticsearch.xpack.core.security.action.user.GetUserPrivilegesResponse;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
-import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
-import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsDefinition;
-import org.elasticsearch.xpack.core.security.authz.permission.IndicesPermission;
-import org.elasticsearch.xpack.core.security.authz.permission.Role;
-import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege;
-import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilege;
-import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivilege;
-import org.elasticsearch.xpack.core.security.authz.privilege.Privilege;
 import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.security.authz.AuthorizationService;
 
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.LinkedHashSet;
-import java.util.Set;
-import java.util.TreeSet;
-
-import static org.elasticsearch.common.Strings.arrayToCommaDelimitedString;
-
 /**
  * Transport action for {@link GetUserPrivilegesAction}
  */
@@ -67,68 +46,6 @@ public class TransportGetUserPrivilegesAction extends HandledTransportAction<Get
             return;
         }
 
-        authorizationService.roles(user, authentication, ActionListener.wrap(
-            role -> listener.onResponse(buildResponseObject(role)),
-            listener::onFailure));
-    }
-
-    // package protected for testing
-    GetUserPrivilegesResponse buildResponseObject(Role userRole) {
-        logger.trace(() -> new ParameterizedMessage("List privileges for role [{}]", arrayToCommaDelimitedString(userRole.names())));
-
-        // We use sorted sets for Strings because they will typically be small, and having a predictable order allows for simpler testing
-        final Set<String> cluster = new TreeSet<>();
-        // But we don't have a meaningful ordering for objects like ConditionalClusterPrivilege, so the tests work with "random" ordering
-        final Set<ConditionalClusterPrivilege> conditionalCluster = new HashSet<>();
-        for (Tuple<ClusterPrivilege, ConditionalClusterPrivilege> tup : userRole.cluster().privileges()) {
-            if (tup.v2() == null) {
-                if (ClusterPrivilege.NONE.equals(tup.v1()) == false) {
-                    cluster.addAll(tup.v1().name());
-                }
-            } else {
-                conditionalCluster.add(tup.v2());
-            }
-        }
-
-        final Set<GetUserPrivilegesResponse.Indices> indices = new LinkedHashSet<>();
-        for (IndicesPermission.Group group : userRole.indices().groups()) {
-            final Set<BytesReference> queries = group.getQuery() == null ? Collections.emptySet() : group.getQuery();
-            final Set<FieldPermissionsDefinition.FieldGrantExcludeGroup> fieldSecurity = group.getFieldPermissions().hasFieldLevelSecurity()
-                ? group.getFieldPermissions().getFieldPermissionsDefinition().getFieldGrantExcludeGroups() : Collections.emptySet();
-            indices.add(new GetUserPrivilegesResponse.Indices(
-                Arrays.asList(group.indices()),
-                group.privilege().name(),
-                fieldSecurity,
-                queries,
-                group.allowRestrictedIndices()
-            ));
-        }
-
-        final Set<RoleDescriptor.ApplicationResourcePrivileges> application = new LinkedHashSet<>();
-        for (String applicationName : userRole.application().getApplicationNames()) {
-            for (ApplicationPrivilege privilege : userRole.application().getPrivileges(applicationName)) {
-                final Set<String> resources = userRole.application().getResourcePatterns(privilege);
-                if (resources.isEmpty()) {
-                    logger.trace("No resources defined in application privilege {}", privilege);
-                } else {
-                    application.add(RoleDescriptor.ApplicationResourcePrivileges.builder()
-                        .application(applicationName)
-                        .privileges(privilege.name())
-                        .resources(resources)
-                        .build());
-                }
-            }
-        }
-
-        final Privilege runAsPrivilege = userRole.runAs().getPrivilege();
-        final Set<String> runAs;
-        if (Operations.isEmpty(runAsPrivilege.getAutomaton())) {
-            runAs = Collections.emptySet();
-        } else {
-            runAs = runAsPrivilege.name();
-        }
-
-        return new GetUserPrivilegesResponse(cluster, conditionalCluster, indices, application, runAs);
+        authorizationService.retrieveUserPrivileges(authentication, request, listener);
     }
-
 }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesAction.java

@@ -5,13 +5,9 @@
  */
 package org.elasticsearch.xpack.security.action.user;
 
-import com.google.common.collect.Sets;
-
-import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -21,20 +17,13 @@ import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesRequest;
 import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesResponse;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
-import org.elasticsearch.xpack.core.security.authz.permission.ResourcePrivileges;
-import org.elasticsearch.xpack.core.security.authz.permission.ResourcePrivilegesMap;
-import org.elasticsearch.xpack.core.security.authz.permission.Role;
 import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
-import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilege;
 import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.security.authz.AuthorizationService;
 import org.elasticsearch.xpack.security.authz.store.NativePrivilegeStore;
 
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -69,10 +58,8 @@ public class TransportHasPrivilegesAction extends HandledTransportAction<HasPriv
             return;
         }
 
-        authorizationService.roles(user, authentication, ActionListener.wrap(
-            role -> resolveApplicationPrivileges(request, ActionListener.wrap(
-                applicationPrivilegeLookup -> checkPrivileges(request, role, applicationPrivilegeLookup, listener),
-                listener::onFailure)),
+        resolveApplicationPrivileges(request, ActionListener.wrap(applicationPrivilegeDescriptors ->
+                authorizationService.checkPrivileges(authentication, request, applicationPrivilegeDescriptors, listener),
             listener::onFailure));
     }
 
@@ -82,56 +69,9 @@ public class TransportHasPrivilegesAction extends HandledTransportAction<HasPriv
         privilegeStore.getPrivileges(applications, null, listener);
     }
 
-    private Set<String> getApplicationNames(HasPrivilegesRequest request) {
+    public static Set<String> getApplicationNames(HasPrivilegesRequest request) {
         return Arrays.stream(request.applicationPrivileges())
             .map(RoleDescriptor.ApplicationResourcePrivileges::getApplication)
             .collect(Collectors.toSet());
     }
-
-    private void checkPrivileges(HasPrivilegesRequest request, Role userRole,
-                                 Collection<ApplicationPrivilegeDescriptor> applicationPrivileges,
-                                 ActionListener<HasPrivilegesResponse> listener) {
-        logger.trace(() -> new ParameterizedMessage("Check whether role [{}] has privileges cluster=[{}] index=[{}] application=[{}]",
-            Strings.arrayToCommaDelimitedString(userRole.names()),
-            Strings.arrayToCommaDelimitedString(request.clusterPrivileges()),
-            Strings.arrayToCommaDelimitedString(request.indexPrivileges()),
-            Strings.arrayToCommaDelimitedString(request.applicationPrivileges())
-        ));
-
-        Map<String, Boolean> cluster = new HashMap<>();
-        for (String checkAction : request.clusterPrivileges()) {
-            final ClusterPrivilege checkPrivilege = ClusterPrivilege.get(Collections.singleton(checkAction));
-            cluster.put(checkAction, userRole.grants(checkPrivilege));
-        }
-        boolean allMatch = cluster.values().stream().allMatch(Boolean::booleanValue);
-
-        ResourcePrivilegesMap.Builder combineIndicesResourcePrivileges = ResourcePrivilegesMap.builder();
-        for (RoleDescriptor.IndicesPrivileges check : request.indexPrivileges()) {
-            ResourcePrivilegesMap resourcePrivileges = userRole.checkIndicesPrivileges(Sets.newHashSet(check.getIndices()),
-                    check.allowRestrictedIndices(), Sets.newHashSet(check.getPrivileges()));
-            allMatch = allMatch && resourcePrivileges.allAllowed();
-            combineIndicesResourcePrivileges.addResourcePrivilegesMap(resourcePrivileges);
-        }
-        ResourcePrivilegesMap allIndices = combineIndicesResourcePrivileges.build();
-        allMatch = allMatch && allIndices.allAllowed();
-
-        final Map<String, Collection<ResourcePrivileges>> privilegesByApplication = new HashMap<>();
-        for (String applicationName : getApplicationNames(request)) {
-            ResourcePrivilegesMap.Builder builder = ResourcePrivilegesMap.builder();
-            for (RoleDescriptor.ApplicationResourcePrivileges p : request.applicationPrivileges()) {
-                if (applicationName.equals(p.getApplication())) {
-                    ResourcePrivilegesMap appPrivsByResourceMap = userRole.checkApplicationResourcePrivileges(applicationName,
-                            Sets.newHashSet(p.getResources()), Sets.newHashSet(p.getPrivileges()), applicationPrivileges);
-                    builder.addResourcePrivilegesMap(appPrivsByResourceMap);
-                }
-            }
-            ResourcePrivilegesMap resourcePrivsForApplication = builder.build();
-            allMatch = allMatch && resourcePrivsForApplication.allAllowed();
-            privilegesByApplication.put(applicationName, resourcePrivsForApplication.getResourceToResourcePrivileges().values());
-        }
-
-        listener.onResponse(new HasPrivilegesResponse(request.username(), allMatch, cluster,
-                allIndices.getResourceToResourcePrivileges().values(), privilegesByApplication));
-    }
-
 }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrail.java

@@ -10,6 +10,7 @@ import org.elasticsearch.transport.TransportMessage;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationToken;
 import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationInfo;
 import org.elasticsearch.xpack.security.transport.filter.SecurityIpFilterRule;
 
 import java.net.InetAddress;
@@ -40,9 +41,11 @@ public interface AuditTrail {
 
     void authenticationFailed(String requestId, String realm, AuthenticationToken token, RestRequest request);
 
-    void accessGranted(String requestId, Authentication authentication, String action, TransportMessage message, String[] roleNames);
+    void accessGranted(String requestId, Authentication authentication, String action, TransportMessage message,
+                       AuthorizationInfo authorizationInfo);
 
-    void accessDenied(String requestId, Authentication authentication, String action, TransportMessage message, String[] roleNames);
+    void accessDenied(String requestId, Authentication authentication, String action, TransportMessage message,
+                      AuthorizationInfo authorizationInfo);
 
     void tamperedRequest(String requestId, RestRequest request);
 
@@ -60,10 +63,13 @@ public interface AuditTrail {
 
     void connectionDenied(InetAddress inetAddress, String profile, SecurityIpFilterRule rule);
 
-    void runAsGranted(String requestId, Authentication authentication, String action, TransportMessage message, String[] roleNames);
+    void runAsGranted(String requestId, Authentication authentication, String action, TransportMessage message,
+                      AuthorizationInfo authorizationInfo);
 
-    void runAsDenied(String requestId, Authentication authentication, String action, TransportMessage message, String[] roleNames);
+    void runAsDenied(String requestId, Authentication authentication, String action, TransportMessage message,
+                     AuthorizationInfo authorizationInfo);
 
-    void runAsDenied(String requestId, Authentication authentication, RestRequest request, String[] roleNames);
+    void runAsDenied(String requestId, Authentication authentication, RestRequest request,
+                     AuthorizationInfo authorizationInfo);
 
 }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrailService.java

@@ -11,6 +11,7 @@ import org.elasticsearch.transport.TransportMessage;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationToken;
 import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationInfo;
 import org.elasticsearch.xpack.security.transport.filter.SecurityIpFilterRule;
 
 import java.net.InetAddress;
@@ -128,19 +129,21 @@ public class AuditTrailService implements AuditTrail {
     }
 
     @Override
-    public void accessGranted(String requestId, Authentication authentication, String action, TransportMessage msg, String[] roleNames) {
+    public void accessGranted(String requestId, Authentication authentication, String action, TransportMessage msg,
+                              AuthorizationInfo authorizationInfo) {
         if (licenseState.isAuditingAllowed()) {
             for (AuditTrail auditTrail : auditTrails) {
-                auditTrail.accessGranted(requestId, authentication, action, msg, roleNames);
+                auditTrail.accessGranted(requestId, authentication, action, msg, authorizationInfo);
             }
         }
     }
 
     @Override
-    public void accessDenied(String requestId, Authentication authentication, String action, TransportMessage message, String[] roleNames) {
+    public void accessDenied(String requestId, Authentication authentication, String action, TransportMessage message,
+                             AuthorizationInfo authorizationInfo) {
         if (licenseState.isAuditingAllowed()) {
             for (AuditTrail auditTrail : auditTrails) {
-                auditTrail.accessDenied(requestId, authentication, action, message, roleNames);
+                auditTrail.accessDenied(requestId, authentication, action, message, authorizationInfo);
             }
         }
     }
@@ -191,28 +194,31 @@ public class AuditTrailService implements AuditTrail {
     }
 
     @Override
-    public void runAsGranted(String requestId, Authentication authentication, String action, TransportMessage message, String[] roleNames) {
+    public void runAsGranted(String requestId, Authentication authentication, String action, TransportMessage message,
+                             AuthorizationInfo authorizationInfo) {
         if (licenseState.isAuditingAllowed()) {
             for (AuditTrail auditTrail : auditTrails) {
-                auditTrail.runAsGranted(requestId, authentication, action, message, roleNames);
+                auditTrail.runAsGranted(requestId, authentication, action, message, authorizationInfo);
             }
         }
     }
 
     @Override
-    public void runAsDenied(String requestId, Authentication authentication, String action, TransportMessage message, String[] roleNames) {
+    public void runAsDenied(String requestId, Authentication authentication, String action, TransportMessage message,
+                            AuthorizationInfo authorizationInfo) {
         if (licenseState.isAuditingAllowed()) {
             for (AuditTrail auditTrail : auditTrails) {
-                auditTrail.runAsDenied(requestId, authentication, action, message, roleNames);
+                auditTrail.runAsDenied(requestId, authentication, action, message, authorizationInfo);
             }
         }
     }
 
     @Override
-    public void runAsDenied(String requestId, Authentication authentication, RestRequest request, String[] roleNames) {
+    public void runAsDenied(String requestId, Authentication authentication, RestRequest request,
+                            AuthorizationInfo authorizationInfo) {
         if (licenseState.isAuditingAllowed()) {
             for (AuditTrail auditTrail : auditTrails) {
-                auditTrail.runAsDenied(requestId, authentication, request, roleNames);
+                auditTrail.runAsDenied(requestId, authentication, request, authorizationInfo);
             }
         }
     }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java

@@ -34,6 +34,7 @@ import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.core.security.user.XPackUser;
 import org.elasticsearch.xpack.security.audit.AuditLevel;
 import org.elasticsearch.xpack.security.audit.AuditTrail;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationInfo;
 import org.elasticsearch.xpack.security.rest.RemoteHostHeader;
 import org.elasticsearch.xpack.security.transport.filter.IPFilter;
 import org.elasticsearch.xpack.security.transport.filter.SecurityIpFilterRule;
@@ -50,6 +51,7 @@ import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.TreeMap;
@@ -415,13 +417,14 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
     }
 
     @Override
-    public void accessGranted(String requestId, Authentication authentication, String action, TransportMessage msg, String[] roleNames) {
+    public void accessGranted(String requestId, Authentication authentication, String action, TransportMessage msg,
+                              AuthorizationInfo authorizationInfo) {
         final User user = authentication.getUser();
         final boolean isSystem = SystemUser.is(user) || XPackUser.is(user);
         if ((isSystem && events.contains(SYSTEM_ACCESS_GRANTED)) || ((isSystem == false) && events.contains(ACCESS_GRANTED))) {
             final Optional<String[]> indices = indices(msg);
             if (eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.of(user),
-                    Optional.of(effectiveRealmName(authentication)), Optional.of(roleNames), indices)) == false) {
+                    Optional.of(effectiveRealmName(authentication)), Optional.of(authorizationInfo), indices)) == false) {
                 final StringMapMessage logEntry = new LogEntryBuilder()
                         .with(EVENT_TYPE_FIELD_NAME, TRANSPORT_ORIGIN_FIELD_VALUE)
                         .with(EVENT_ACTION_FIELD_NAME, "access_granted")
@@ -431,9 +434,9 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
                         .withSubject(authentication)
                         .withRestOrTransportOrigin(msg, threadContext)
                         .with(INDICES_FIELD_NAME, indices.orElse(null))
-                        .with(PRINCIPAL_ROLES_FIELD_NAME, roleNames)
                         .withOpaqueId(threadContext)
                         .withXForwardedFor(threadContext)
+                        .with(authorizationInfo.asMap())
                         .build();
                 logger.info(logEntry);
             }
@@ -441,11 +444,12 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
     }
 
     @Override
-    public void accessDenied(String requestId, Authentication authentication, String action, TransportMessage message, String[] roleNames) {
+    public void accessDenied(String requestId, Authentication authentication, String action, TransportMessage message,
+                             AuthorizationInfo authorizationInfo) {
         if (events.contains(ACCESS_DENIED)) {
             final Optional<String[]> indices = indices(message);
             if (eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.of(authentication.getUser()),
-                    Optional.of(effectiveRealmName(authentication)), Optional.of(roleNames), indices)) == false) {
+                    Optional.of(effectiveRealmName(authentication)), Optional.of(authorizationInfo), indices)) == false) {
                 final StringMapMessage logEntry = new LogEntryBuilder()
                         .with(EVENT_TYPE_FIELD_NAME, TRANSPORT_ORIGIN_FIELD_VALUE)
                         .with(EVENT_ACTION_FIELD_NAME, "access_denied")
@@ -455,7 +459,7 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
                         .withSubject(authentication)
                         .withRestOrTransportOrigin(message, threadContext)
                         .with(INDICES_FIELD_NAME, indices.orElse(null))
-                        .with(PRINCIPAL_ROLES_FIELD_NAME, roleNames)
+                        .with(authorizationInfo.asMap())
                         .withOpaqueId(threadContext)
                         .withXForwardedFor(threadContext)
                         .build();
@@ -563,11 +567,12 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
     }
 
     @Override
-    public void runAsGranted(String requestId, Authentication authentication, String action, TransportMessage message, String[] roleNames) {
+    public void runAsGranted(String requestId, Authentication authentication, String action, TransportMessage message,
+                             AuthorizationInfo authorizationInfo) {
         if (events.contains(RUN_AS_GRANTED)) {
             final Optional<String[]> indices = indices(message);
             if (eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.of(authentication.getUser()),
-                    Optional.of(effectiveRealmName(authentication)), Optional.of(roleNames), indices)) == false) {
+                    Optional.of(effectiveRealmName(authentication)), Optional.of(authorizationInfo), indices)) == false) {
                 final StringMapMessage logEntry = new LogEntryBuilder()
                         .with(EVENT_TYPE_FIELD_NAME, TRANSPORT_ORIGIN_FIELD_VALUE)
                         .with(EVENT_ACTION_FIELD_NAME, "run_as_granted")
@@ -577,7 +582,7 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
                         .withRunAsSubject(authentication)
                         .withRestOrTransportOrigin(message, threadContext)
                         .with(INDICES_FIELD_NAME, indices.orElse(null))
-                        .with(PRINCIPAL_ROLES_FIELD_NAME, roleNames)
+                        .with(authorizationInfo.asMap())
                         .withOpaqueId(threadContext)
                         .withXForwardedFor(threadContext)
                         .build();
@@ -587,11 +592,12 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
     }
 
     @Override
-    public void runAsDenied(String requestId, Authentication authentication, String action, TransportMessage message, String[] roleNames) {
+    public void runAsDenied(String requestId, Authentication authentication, String action, TransportMessage message,
+                            AuthorizationInfo authorizationInfo) {
         if (events.contains(RUN_AS_DENIED)) {
             final Optional<String[]> indices = indices(message);
             if (eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.of(authentication.getUser()),
-                    Optional.of(effectiveRealmName(authentication)), Optional.of(roleNames), indices)) == false) {
+                    Optional.of(effectiveRealmName(authentication)), Optional.of(authorizationInfo), indices)) == false) {
                 final StringMapMessage logEntry = new LogEntryBuilder()
                         .with(EVENT_TYPE_FIELD_NAME, TRANSPORT_ORIGIN_FIELD_VALUE)
                         .with(EVENT_ACTION_FIELD_NAME, "run_as_denied")
@@ -601,7 +607,7 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
                         .withRunAsSubject(authentication)
                         .withRestOrTransportOrigin(message, threadContext)
                         .with(INDICES_FIELD_NAME, indices.orElse(null))
-                        .with(PRINCIPAL_ROLES_FIELD_NAME, roleNames)
+                        .with(authorizationInfo.asMap())
                         .withOpaqueId(threadContext)
                         .withXForwardedFor(threadContext)
                         .build();
@@ -611,14 +617,14 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
     }
 
     @Override
-    public void runAsDenied(String requestId, Authentication authentication, RestRequest request, String[] roleNames) {
+    public void runAsDenied(String requestId, Authentication authentication, RestRequest request, AuthorizationInfo authorizationInfo) {
         if (events.contains(RUN_AS_DENIED)
                 && eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.of(authentication.getUser()),
-                        Optional.of(effectiveRealmName(authentication)), Optional.of(roleNames), Optional.empty())) == false) {
+                        Optional.of(effectiveRealmName(authentication)), Optional.of(authorizationInfo), Optional.empty())) == false) {
             final StringMapMessage logEntry = new LogEntryBuilder()
                     .with(EVENT_TYPE_FIELD_NAME, REST_ORIGIN_FIELD_VALUE)
                     .with(EVENT_ACTION_FIELD_NAME, "run_as_denied")
-                    .with(PRINCIPAL_ROLES_FIELD_NAME, roleNames)
+                    .with(authorizationInfo.asMap())
                     .withRestUriAndMethod(request)
                     .withRunAsSubject(authentication)
                     .withRestOrigin(request)
@@ -762,29 +768,40 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
             return this;
         }
 
+        LogEntryBuilder with(Map<String, Object> map) {
+            for (Entry<String, Object> entry : map.entrySet()) {
+                Object value = entry.getValue();
+                if (value.getClass().isArray()) {
+                    logEntry.with(entry.getKey(), toQuotedJsonArray((Object[]) value));
+                } else {
+                    logEntry.with(entry.getKey(), value);
+                }
+            }
+            return this;
+        }
+
         StringMapMessage build() {
             return logEntry;
         }
 
-        String toQuotedJsonArray(String[] values) {
+        String toQuotedJsonArray(Object[] values) {
             assert values != null;
             final StringBuilder stringBuilder = new StringBuilder();
             final JsonStringEncoder jsonStringEncoder = JsonStringEncoder.getInstance();
             stringBuilder.append("[");
-            for (final String value : values) {
+            for (final Object value : values) {
                 if (value != null) {
                     if (stringBuilder.length() > 1) {
                         stringBuilder.append(",");
                     }
                     stringBuilder.append("\"");
-                    jsonStringEncoder.quoteAsString(value, stringBuilder);
+                    jsonStringEncoder.quoteAsString(value.toString(), stringBuilder);
                     stringBuilder.append("\"");
                 }
             }
             stringBuilder.append("]");
             return stringBuilder.toString();
         }
-
     }
 
 
@@ -979,7 +996,8 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
          * user field (such as `anonymous_access_denied`) as well as events from the
          * "elastic" username.
          */
-        AuditEventMetaInfo(Optional<User> user, Optional<String> realm, Optional<String[]> roles, Optional<String[]> indices) {
+        AuditEventMetaInfo(Optional<User> user, Optional<String> realm, Optional<AuthorizationInfo> authorizationInfo,
+                           Optional<String[]> indices) {
             this.principal = user.map(u -> u.principal()).orElse("");
             this.realm = realm.orElse("");
             // Supplier indirection and lazy generation of Streams serves 2 purposes:
@@ -987,8 +1005,12 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
             // conditions on the `principal` and `realm` fields
             // 2. reusability of the AuditEventMetaInfo instance: in this case Streams have
             // to be regenerated as they cannot be operated upon twice
-            this.roles = () -> roles.filter(r -> r.length > 0).filter(a -> Arrays.stream(a).anyMatch(Objects::nonNull))
-                    .map(Arrays::stream).orElse(Stream.of(""));
+            this.roles = () -> authorizationInfo.filter(info -> {
+                final Object value = info.asMap().get("user.roles");
+                return value instanceof String[] &&
+                    ((String[]) value).length != 0 &&
+                    Arrays.stream((String[]) value).anyMatch(Objects::nonNull);
+            }).map(info -> Arrays.stream((String[]) info.asMap().get("user.roles"))).orElse(Stream.of(""));
             this.indices = () -> indices.filter(i -> i.length > 0).filter(a -> Arrays.stream(a).anyMatch(Objects::nonNull))
                     .map(Arrays::stream).orElse(Stream.of(""));
         }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java

@@ -6,8 +6,6 @@
 
 package org.elasticsearch.xpack.security.authc;
 
-import com.google.common.collect.Sets;
-
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
@@ -62,12 +60,10 @@ import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
 import org.elasticsearch.xpack.core.security.authc.support.Hasher;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
-import org.elasticsearch.xpack.core.security.authz.permission.Role;
-import org.elasticsearch.xpack.core.security.authz.permission.LimitedRole;
 import org.elasticsearch.xpack.core.security.user.User;
-import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 
+import javax.crypto.SecretKeyFactory;
 import java.io.Closeable;
 import java.io.IOException;
 import java.io.UncheckedIOException;
@@ -84,11 +80,10 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
-import javax.crypto.SecretKeyFactory;
-
 import static org.elasticsearch.search.SearchService.DEFAULT_KEEPALIVE_SETTING;
 import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_ORIGIN;
 import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
@@ -132,12 +127,11 @@ public class ApiKeyService {
     private final Settings settings;
     private final ExpiredApiKeysRemover expiredApiKeysRemover;
     private final TimeValue deleteInterval;
-    private final CompositeRolesStore compositeRolesStore;
 
     private volatile long lastExpirationRunMs;
 
-    public ApiKeyService(Settings settings, Clock clock, Client client, SecurityIndexManager securityIndex, ClusterService clusterService,
-            CompositeRolesStore compositeRolesStore) {
+    public ApiKeyService(Settings settings, Clock clock, Client client, SecurityIndexManager securityIndex,
+                         ClusterService clusterService) {
         this.clock = clock;
         this.client = client;
         this.securityIndex = securityIndex;
@@ -147,16 +141,17 @@ public class ApiKeyService {
         this.settings = settings;
         this.deleteInterval = DELETE_INTERVAL.get(settings);
         this.expiredApiKeysRemover = new ExpiredApiKeysRemover(settings, client);
-        this.compositeRolesStore = compositeRolesStore;
     }
 
     /**
      * Asynchronously creates a new API key based off of the request and authentication
      * @param authentication the authentication that this api key should be based off of
      * @param request the request to create the api key included any permission restrictions
+     * @param roleDescriptorSet the user's actual roles that we always enforce
      * @param listener the listener that will be used to notify of completion
      */
-    public void createApiKey(Authentication authentication, CreateApiKeyRequest request, ActionListener<CreateApiKeyResponse> listener) {
+    public void createApiKey(Authentication authentication, CreateApiKeyRequest request, Set<RoleDescriptor> roleDescriptorSet,
+                             ActionListener<CreateApiKeyResponse> listener) {
         ensureEnabled();
         if (authentication == null) {
             listener.onFailure(new IllegalArgumentException("authentication must be provided"));
@@ -209,13 +204,10 @@ public class ApiKeyService {
 
                         // Save limited_by_role_descriptors
                         builder.startObject("limited_by_role_descriptors");
-                        compositeRolesStore.getRoleDescriptors(Sets.newHashSet(authentication.getUser().roles()),
-                                ActionListener.wrap(rdSet -> {
-                                    for (RoleDescriptor descriptor : rdSet) {
-                                        builder.field(descriptor.getName(),
-                                                (contentBuilder, params) -> descriptor.toXContent(contentBuilder, params, true));
-                                    }
-                                }, listener::onFailure));
+                        for (RoleDescriptor descriptor : roleDescriptorSet) {
+                            builder.field(descriptor.getName(),
+                                    (contentBuilder, params) -> descriptor.toXContent(contentBuilder, params, true));
+                        }
                         builder.endObject();
 
                         builder.field("name", request.getName())
@@ -294,10 +286,9 @@ public class ApiKeyService {
 
     /**
      * The current request has been authenticated by an API key and this method enables the
-     * retrieval of role descriptors that are associated with the api key and triggers the building
-     * of the {@link Role} to authorize the request.
+     * retrieval of role descriptors that are associated with the api key
      */
-    public void getRoleForApiKey(Authentication authentication, CompositeRolesStore rolesStore, ActionListener<Role> listener) {
+    public void getRoleForApiKey(Authentication authentication, ActionListener<ApiKeyRoleDescriptors> listener) {
         if (authentication.getAuthenticationType() != Authentication.AuthenticationType.API_KEY) {
             throw new IllegalStateException("authentication type must be api key but is " + authentication.getAuthenticationType());
         }
@@ -312,18 +303,37 @@ public class ApiKeyService {
             listener.onFailure(new ElasticsearchSecurityException("no role descriptors found for API key"));
         } else if (roleDescriptors == null || roleDescriptors.isEmpty()) {
             final List<RoleDescriptor> authnRoleDescriptorsList = parseRoleDescriptors(apiKeyId, authnRoleDescriptors);
-            rolesStore.buildAndCacheRoleFromDescriptors(authnRoleDescriptorsList, apiKeyId, listener);
+            listener.onResponse(new ApiKeyRoleDescriptors(apiKeyId, authnRoleDescriptorsList, null));
         } else {
             final List<RoleDescriptor> roleDescriptorList = parseRoleDescriptors(apiKeyId, roleDescriptors);
             final List<RoleDescriptor> authnRoleDescriptorsList = parseRoleDescriptors(apiKeyId, authnRoleDescriptors);
-            rolesStore.buildAndCacheRoleFromDescriptors(roleDescriptorList, apiKeyId, ActionListener.wrap(role -> {
-                rolesStore.buildAndCacheRoleFromDescriptors(authnRoleDescriptorsList, apiKeyId, ActionListener.wrap(limitedByRole -> {
-                    Role finalRole = LimitedRole.createLimitedRole(role, limitedByRole);
-                    listener.onResponse(finalRole);
-                }, listener::onFailure));
-            }, listener::onFailure));
+            listener.onResponse(new ApiKeyRoleDescriptors(apiKeyId, roleDescriptorList, authnRoleDescriptorsList));
         }
+    }
 
+    public static class ApiKeyRoleDescriptors {
+
+        private final String apiKeyId;
+        private final List<RoleDescriptor> roleDescriptors;
+        private final List<RoleDescriptor> limitedByRoleDescriptors;
+
+        public ApiKeyRoleDescriptors(String apiKeyId, List<RoleDescriptor> roleDescriptors, List<RoleDescriptor> limitedByDescriptors) {
+            this.apiKeyId = apiKeyId;
+            this.roleDescriptors = roleDescriptors;
+            this.limitedByRoleDescriptors = limitedByDescriptors;
+        }
+
+        public String getApiKeyId() {
+            return apiKeyId;
+        }
+
+        public List<RoleDescriptor> getRoleDescriptors() {
+            return roleDescriptors;
+        }
+
+        public List<RoleDescriptor> getLimitedByRoleDescriptors() {
+            return limitedByRoleDescriptors;
+        }
     }
 
     private List<RoleDescriptor> parseRoleDescriptors(final String apiKeyId, final Map<String, Object> roleDescriptors) {

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java

@@ -35,7 +35,6 @@ import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationServiceField;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationToken;
 import org.elasticsearch.xpack.core.security.authc.Realm;
-import org.elasticsearch.xpack.core.security.authz.permission.Role;
 import org.elasticsearch.xpack.core.security.support.Exceptions;
 import org.elasticsearch.xpack.core.security.user.AnonymousUser;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
@@ -44,6 +43,7 @@ import org.elasticsearch.xpack.security.audit.AuditTrail;
 import org.elasticsearch.xpack.security.audit.AuditTrailService;
 import org.elasticsearch.xpack.security.audit.AuditUtil;
 import org.elasticsearch.xpack.security.authc.support.RealmUserLookup;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.EmptyAuthorizationInfo;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 
 import java.util.ArrayList;
@@ -643,7 +643,7 @@ public class AuthenticationService {
 
         @Override
         ElasticsearchSecurityException runAsDenied(Authentication authentication, AuthenticationToken token) {
-            auditTrail.runAsDenied(requestId, authentication, action, message, Role.EMPTY.names());
+            auditTrail.runAsDenied(requestId, authentication, action, message, EmptyAuthorizationInfo.INSTANCE);
             return failureHandler.failedAuthentication(message, token, action, threadContext);
         }
 
@@ -707,7 +707,7 @@ public class AuthenticationService {
 
         @Override
         ElasticsearchSecurityException runAsDenied(Authentication authentication, AuthenticationToken token) {
-            auditTrail.runAsDenied(requestId, authentication, request, Role.EMPTY.names());
+            auditTrail.runAsDenied(requestId, authentication, request, EmptyAuthorizationInfo.INSTANCE);
             return failureHandler.failedAuthentication(request, token, threadContext);
         }
 

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java

@@ -3,59 +3,59 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
+
 package org.elasticsearch.xpack.security.authz;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.CompositeIndicesRequest;
 import org.elasticsearch.action.DocWriteRequest;
-import org.elasticsearch.action.IndicesRequest;
+import org.elasticsearch.action.StepListener;
 import org.elasticsearch.action.admin.indices.alias.Alias;
-import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesAction;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
-import org.elasticsearch.action.bulk.BulkAction;
 import org.elasticsearch.action.bulk.BulkItemRequest;
 import org.elasticsearch.action.bulk.BulkShardRequest;
 import org.elasticsearch.action.bulk.TransportShardBulkAction;
 import org.elasticsearch.action.delete.DeleteAction;
-import org.elasticsearch.action.get.MultiGetAction;
 import org.elasticsearch.action.index.IndexAction;
-import org.elasticsearch.action.search.ClearScrollAction;
-import org.elasticsearch.action.search.MultiSearchAction;
-import org.elasticsearch.action.search.SearchScrollAction;
-import org.elasticsearch.action.search.SearchTransportService;
+import org.elasticsearch.action.support.GroupedActionListener;
 import org.elasticsearch.action.support.replication.TransportReplicationAction.ConcreteShardRequest;
-import org.elasticsearch.action.termvectors.MultiTermVectorsAction;
 import org.elasticsearch.action.update.UpdateAction;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportActionProxy;
 import org.elasticsearch.transport.TransportRequest;
-import org.elasticsearch.xpack.core.security.action.user.AuthenticateAction;
-import org.elasticsearch.xpack.core.security.action.user.ChangePasswordAction;
-import org.elasticsearch.xpack.core.security.action.user.GetUserPrivilegesAction;
-import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesAction;
-import org.elasticsearch.xpack.core.security.action.user.UserRequest;
+import org.elasticsearch.xpack.core.security.action.user.GetUserPrivilegesRequest;
+import org.elasticsearch.xpack.core.security.action.user.GetUserPrivilegesResponse;
+import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesRequest;
+import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesResponse;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationFailureHandler;
-import org.elasticsearch.xpack.core.security.authc.esnative.NativeRealmSettings;
+import org.elasticsearch.xpack.core.security.authc.esnative.ClientReservedRealm;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AsyncSupplier;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationInfo;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationResult;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.EmptyAuthorizationInfo;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.IndexAuthorizationResult;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.RequestInfo;
 import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField;
+import org.elasticsearch.xpack.core.security.authz.ResolvedIndices;
 import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
-import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsCache;
-import org.elasticsearch.xpack.core.security.authz.permission.Role;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
 import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilege;
 import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege;
-import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
-import org.elasticsearch.xpack.core.security.support.Automatons;
 import org.elasticsearch.xpack.core.security.user.AnonymousUser;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
 import org.elasticsearch.xpack.core.security.user.User;
@@ -63,54 +63,55 @@ import org.elasticsearch.xpack.core.security.user.XPackSecurityUser;
 import org.elasticsearch.xpack.core.security.user.XPackUser;
 import org.elasticsearch.xpack.security.audit.AuditTrailService;
 import org.elasticsearch.xpack.security.audit.AuditUtil;
-import org.elasticsearch.xpack.security.authc.ApiKeyService;
-import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm;
-import org.elasticsearch.xpack.security.authz.IndicesAndAliasesResolver.ResolvedIndices;
+import org.elasticsearch.xpack.security.authz.interceptor.RequestInterceptor;
 import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore;
 
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
-import java.util.function.Predicate;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Consumer;
 
+import static org.elasticsearch.action.support.ContextPreservingActionListener.wrapPreservingContext;
 import static org.elasticsearch.xpack.core.security.SecurityField.setting;
 import static org.elasticsearch.xpack.core.security.support.Exceptions.authorizationError;
+import static org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail.PRINCIPAL_ROLES_FIELD_NAME;
 
 public class AuthorizationService {
     public static final Setting<Boolean> ANONYMOUS_AUTHORIZATION_EXCEPTION_SETTING =
         Setting.boolSetting(setting("authc.anonymous.authz_exception"), true, Property.NodeScope);
     public static final String ORIGINATING_ACTION_KEY = "_originating_action_name";
-    public static final String ROLE_NAMES_KEY = "_effective_role_names";
-
-    private static final Predicate<String> SAME_USER_PRIVILEGE = Automatons.predicate(
-            ChangePasswordAction.NAME, AuthenticateAction.NAME, HasPrivilegesAction.NAME, GetUserPrivilegesAction.NAME);
+    public static final String AUTHORIZATION_INFO_KEY = "_authz_info";
+    private static final AuthorizationInfo SYSTEM_AUTHZ_INFO =
+        () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, new String[] { SystemUser.ROLE_NAME });
 
-    private static final String INDEX_SUB_REQUEST_PRIMARY = IndexAction.NAME + "[p]";
-    private static final String INDEX_SUB_REQUEST_REPLICA = IndexAction.NAME + "[r]";
-    private static final String DELETE_SUB_REQUEST_PRIMARY = DeleteAction.NAME + "[p]";
-    private static final String DELETE_SUB_REQUEST_REPLICA = DeleteAction.NAME + "[r]";
     private static final Logger logger = LogManager.getLogger(AuthorizationService.class);
 
+    private final Settings settings;
     private final ClusterService clusterService;
-    private final CompositeRolesStore rolesStore;
     private final AuditTrailService auditTrail;
     private final IndicesAndAliasesResolver indicesAndAliasesResolver;
     private final AuthenticationFailureHandler authcFailureHandler;
     private final ThreadContext threadContext;
     private final AnonymousUser anonymousUser;
-    private final FieldPermissionsCache fieldPermissionsCache;
-    private final ApiKeyService apiKeyService;
+    private final AuthorizationEngine rbacEngine;
+    private final AuthorizationEngine authorizationEngine;
+    private final Set<RequestInterceptor> requestInterceptors;
+    private final XPackLicenseState licenseState;
     private final boolean isAnonymousEnabled;
     private final boolean anonymousAuthzExceptionEnabled;
 
     public AuthorizationService(Settings settings, CompositeRolesStore rolesStore, ClusterService clusterService,
                                 AuditTrailService auditTrail, AuthenticationFailureHandler authcFailureHandler,
-                                ThreadPool threadPool, AnonymousUser anonymousUser, ApiKeyService apiKeyService,
-                                FieldPermissionsCache fieldPermissionsCache) {
-        this.rolesStore = rolesStore;
+                                ThreadPool threadPool, AnonymousUser anonymousUser, @Nullable AuthorizationEngine authorizationEngine,
+                                Set<RequestInterceptor> requestInterceptors, XPackLicenseState licenseState) {
         this.clusterService = clusterService;
         this.auditTrail = auditTrail;
         this.indicesAndAliasesResolver = new IndicesAndAliasesResolver(settings, clusterService);
@@ -119,8 +120,27 @@ public class AuthorizationService {
         this.anonymousUser = anonymousUser;
         this.isAnonymousEnabled = AnonymousUser.isAnonymousEnabled(settings);
         this.anonymousAuthzExceptionEnabled = ANONYMOUS_AUTHORIZATION_EXCEPTION_SETTING.get(settings);
-        this.fieldPermissionsCache = fieldPermissionsCache;
-        this.apiKeyService = apiKeyService;
+        this.rbacEngine = new RBACEngine(settings, rolesStore);
+        this.authorizationEngine = authorizationEngine == null ? this.rbacEngine : authorizationEngine;
+        this.requestInterceptors = requestInterceptors;
+        this.settings = settings;
+        this.licenseState = licenseState;
+    }
+
+    public void checkPrivileges(Authentication authentication, HasPrivilegesRequest request,
+                                Collection<ApplicationPrivilegeDescriptor> applicationPrivilegeDescriptors,
+                                ActionListener<HasPrivilegesResponse> listener) {
+        getAuthorizationEngine(authentication).checkPrivileges(authentication, getAuthorizationInfoFromContext(), request,
+            applicationPrivilegeDescriptors, wrapPreservingContext(listener, threadContext));
+    }
+
+    public void retrieveUserPrivileges(Authentication authentication, GetUserPrivilegesRequest request,
+                                       ActionListener<GetUserPrivilegesResponse> listener) {
+        getAuthorizationEngine(authentication).getUserPrivileges(authentication, getAuthorizationInfoFromContext(), request, listener);
+    }
+
+    private AuthorizationInfo getAuthorizationInfoFromContext() {
+        return Objects.requireNonNull(threadContext.getTransient(AUTHORIZATION_INFO_KEY), "authorization info is missing from context");
     }
 
     /**
@@ -128,14 +148,16 @@ public class AuthorizationService {
      * have the appropriate privileges for this action/request, an {@link ElasticsearchSecurityException}
      * will be thrown.
      *
-     * @param authentication The authentication information
-     * @param action         The action
-     * @param request        The request
+     * @param authentication  The authentication information
+     * @param action          The action
+     * @param originalRequest The request
+     * @param listener        The listener that gets called. A call to {@link ActionListener#onResponse(Object)} indicates success
      * @throws ElasticsearchSecurityException If the given user is no allowed to execute the given request
      */
-    public void authorize(Authentication authentication, String action, TransportRequest request, Role userRole,
-                          Role runAsRole) throws ElasticsearchSecurityException {
-        final TransportRequest originalRequest = request;
+    public void authorize(final Authentication authentication, final String action, final TransportRequest originalRequest,
+                          final ActionListener<Void> listener) throws ElasticsearchSecurityException {
+        // prior to doing any authorization lets set the originating action in the context only
+        putTransientIfNonExisting(ORIGINATING_ACTION_KEY, action);
 
         String auditId = AuditUtil.extractRequestId(threadContext);
         if (auditId == null) {
@@ -144,212 +166,259 @@ public class AuthorizationService {
             if (isInternalUser(authentication.getUser()) != false) {
                 auditId = AuditUtil.getOrGenerateRequestId(threadContext);
             } else {
-                auditTrail.tamperedRequest(null, authentication.getUser(), action, request);
+                auditTrail.tamperedRequest(null, authentication.getUser(), action, originalRequest);
                 final String message = "Attempt to authorize action [" + action + "] for [" + authentication.getUser().principal()
                     + "] without an existing request-id";
                 assert false : message;
-                throw new ElasticsearchSecurityException(message);
-            }
-        }
-
-        if (request instanceof ConcreteShardRequest) {
-            request = ((ConcreteShardRequest<?>) request).getRequest();
-            assert TransportActionProxy.isProxyRequest(request) == false : "expected non-proxy request for action: " + action;
-        } else {
-            request = TransportActionProxy.unwrapRequest(request);
-            if (TransportActionProxy.isProxyRequest(originalRequest) && TransportActionProxy.isProxyAction(action) == false) {
-                throw new IllegalStateException("originalRequest is a proxy request for: [" + request + "] but action: ["
-                    + action + "] isn't");
+                listener.onFailure(new ElasticsearchSecurityException(message));
             }
         }
-        // prior to doing any authorization lets set the originating action in the context only
-        putTransientIfNonExisting(ORIGINATING_ACTION_KEY, action);
 
-        // first we need to check if the user is the system. If it is, we'll just authorize the system access
+        // sometimes a request might be wrapped within another, which is the case for proxied
+        // requests and concrete shard requests
+        final TransportRequest unwrappedRequest = maybeUnwrapRequest(authentication, originalRequest, action, auditId);
         if (SystemUser.is(authentication.getUser())) {
-            if (SystemUser.isAuthorized(action)) {
-                putTransientIfNonExisting(AuthorizationServiceField.INDICES_PERMISSIONS_KEY, IndicesAccessControl.ALLOW_ALL);
-                putTransientIfNonExisting(ROLE_NAMES_KEY, new String[] { SystemUser.ROLE_NAME });
-                auditTrail.accessGranted(auditId, authentication, action, request, new String[] { SystemUser.ROLE_NAME });
-                return;
-            }
-            throw denial(auditId, authentication, action, request, new String[] { SystemUser.ROLE_NAME });
+            // this never goes async so no need to wrap the listener
+            authorizeSystemUser(authentication, action, auditId, unwrappedRequest, listener);
+        } else {
+            final String finalAuditId = auditId;
+            final RequestInfo requestInfo = new RequestInfo(authentication, unwrappedRequest, action);
+            final ActionListener<AuthorizationInfo> authzInfoListener = wrapPreservingContext(ActionListener.wrap(
+                authorizationInfo -> {
+                    putTransientIfNonExisting(AUTHORIZATION_INFO_KEY, authorizationInfo);
+                    maybeAuthorizeRunAs(requestInfo, finalAuditId, authorizationInfo, listener);
+                }, listener::onFailure), threadContext);
+            getAuthorizationEngine(authentication).resolveAuthorizationInfo(requestInfo, authzInfoListener);
         }
+    }
 
-        // get the roles of the authenticated user, which may be different than the effective
-        Role permission = userRole;
-
-        // check if the request is a run as request
+    private void maybeAuthorizeRunAs(final RequestInfo requestInfo, final String requestId, final AuthorizationInfo authzInfo,
+                                     final ActionListener<Void> listener) {
+        final Authentication authentication = requestInfo.getAuthentication();
+        final TransportRequest request = requestInfo.getRequest();
+        final String action = requestInfo.getAction();
         final boolean isRunAs = authentication.getUser().isRunAs();
         if (isRunAs) {
-            // if we are running as a user we looked up then the authentication must contain a lookedUpBy. If it doesn't then this user
-            // doesn't really exist but the authc service allowed it through to avoid leaking users that exist in the system
-            if (authentication.getLookedUpBy() == null) {
-                throw denyRunAs(auditId, authentication, action, request, permission.names());
-            } else if (permission.runAs().check(authentication.getUser().principal())) {
-                auditTrail.runAsGranted(auditId, authentication, action, request, permission.names());
-                permission = runAsRole;
-            } else {
-                throw denyRunAs(auditId, authentication, action, request, permission.names());
-            }
+            ActionListener<AuthorizationResult> runAsListener = wrapPreservingContext(ActionListener.wrap(result -> {
+                if (result.isGranted()) {
+                    if (result.isAuditable()) {
+                        auditTrail.runAsGranted(requestId, authentication, action, request,
+                            authzInfo.getAuthenticatedUserAuthorizationInfo());
+                    }
+                    authorizeAction(requestInfo, requestId, authzInfo, listener);
+                } else {
+                    if (result.isAuditable()) {
+                        auditTrail.runAsDenied(requestId, authentication, action, request,
+                            authzInfo.getAuthenticatedUserAuthorizationInfo());
+                    }
+                    listener.onFailure(denialException(authentication, action, null));
+                }
+            }, e -> {
+                auditTrail.runAsDenied(requestId, authentication, action, request,
+                    authzInfo.getAuthenticatedUserAuthorizationInfo());
+                listener.onFailure(denialException(authentication, action, null));
+            }), threadContext);
+            authorizeRunAs(requestInfo, authzInfo, runAsListener);
+        } else {
+            authorizeAction(requestInfo, requestId, authzInfo, listener);
         }
-        putTransientIfNonExisting(ROLE_NAMES_KEY, permission.names());
+    }
 
-        // first, we'll check if the action is a cluster action. If it is, we'll only check it against the cluster permissions
+    private void authorizeAction(final RequestInfo requestInfo, final String requestId, final AuthorizationInfo authzInfo,
+                                 final ActionListener<Void> listener) {
+        final Authentication authentication = requestInfo.getAuthentication();
+        final TransportRequest request = requestInfo.getRequest();
+        final String action = requestInfo.getAction();
+        final AuthorizationEngine authzEngine = getAuthorizationEngine(authentication);
         if (ClusterPrivilege.ACTION_MATCHER.test(action)) {
-            if (permission.checkClusterAction(action, request) || checkSameUserPermissions(action, request, authentication)) {
-                putTransientIfNonExisting(AuthorizationServiceField.INDICES_PERMISSIONS_KEY, IndicesAccessControl.ALLOW_ALL);
-                auditTrail.accessGranted(auditId, authentication, action, request, permission.names());
-                return;
-            }
-            throw denial(auditId, authentication, action, request, permission.names());
+            final ActionListener<AuthorizationResult> clusterAuthzListener =
+                wrapPreservingContext(new AuthorizationResultListener<>(result -> {
+                        putTransientIfNonExisting(AuthorizationServiceField.INDICES_PERMISSIONS_KEY, IndicesAccessControl.ALLOW_ALL);
+                        listener.onResponse(null);
+                    }, listener::onFailure, requestInfo, requestId, authzInfo), threadContext);
+            authzEngine.authorizeClusterAction(requestInfo, authzInfo, clusterAuthzListener);
+        } else if (IndexPrivilege.ACTION_MATCHER.test(action)) {
+            final MetaData metaData = clusterService.state().metaData();
+            final AsyncSupplier<List<String>> authorizedIndicesSupplier = new CachingAsyncSupplier<>(authzIndicesListener ->
+                authzEngine.loadAuthorizedIndices(requestInfo, authzInfo, metaData.getAliasAndIndexLookup(),
+                    authzIndicesListener));
+            final AsyncSupplier<ResolvedIndices> resolvedIndicesAsyncSupplier = new CachingAsyncSupplier<>((resolvedIndicesListener) -> {
+                authorizedIndicesSupplier.getAsync(ActionListener.wrap(authorizedIndices -> {
+                    resolveIndexNames(request, metaData, authorizedIndices, resolvedIndicesListener);
+                }, e -> {
+                    auditTrail.accessDenied(requestId, authentication, action, request, authzInfo);
+                    if (e instanceof IndexNotFoundException) {
+                        listener.onFailure(e);
+                    } else {
+                        listener.onFailure(denialException(authentication, action, e));
+                    }
+                }));
+            });
+            authzEngine.authorizeIndexAction(requestInfo, authzInfo, resolvedIndicesAsyncSupplier,
+                metaData.getAliasAndIndexLookup(), wrapPreservingContext(new AuthorizationResultListener<>(result ->
+                    handleIndexActionAuthorizationResult(result, requestInfo, requestId, authzInfo, authzEngine, authorizedIndicesSupplier,
+                        resolvedIndicesAsyncSupplier, metaData, listener),
+                    listener::onFailure, requestInfo, requestId, authzInfo), threadContext));
+        } else {
+            logger.warn("denying access as action [{}] is not an index or cluster action", action);
+            auditTrail.accessDenied(requestId, authentication, action, request, authzInfo);
+            listener.onFailure(denialException(authentication, action, null));
         }
+    }
 
-        // ok... this is not a cluster action, let's verify it's an indices action
-        if (!IndexPrivilege.ACTION_MATCHER.test(action)) {
-            throw denial(auditId, authentication, action, request, permission.names());
+    private void handleIndexActionAuthorizationResult(final IndexAuthorizationResult result, final RequestInfo requestInfo,
+                                                      final String requestId, final AuthorizationInfo authzInfo,
+                                                      final AuthorizationEngine authzEngine,
+                                                      final AsyncSupplier<List<String>> authorizedIndicesSupplier,
+                                                      final AsyncSupplier<ResolvedIndices> resolvedIndicesAsyncSupplier,
+                                                      final MetaData metaData,
+                                                      final ActionListener<Void> listener) {
+        final Authentication authentication = requestInfo.getAuthentication();
+        final TransportRequest request = requestInfo.getRequest();
+        final String action = requestInfo.getAction();
+        if (result.getIndicesAccessControl() != null) {
+            putTransientIfNonExisting(AuthorizationServiceField.INDICES_PERMISSIONS_KEY,
+                result.getIndicesAccessControl());
         }
-
-        //composite actions are explicitly listed and will be authorized at the sub-request / shard level
-        if (isCompositeAction(action)) {
-            if (request instanceof CompositeIndicesRequest == false) {
-                throw new IllegalStateException("Composite actions must implement " + CompositeIndicesRequest.class.getSimpleName()
-                    + ", " + request.getClass().getSimpleName() + " doesn't");
-            }
-            // we check if the user can execute the action, without looking at indices, which will be authorized at the shard level
-            if (permission.checkIndicesAction(action)) {
-                auditTrail.accessGranted(auditId, authentication, action, request, permission.names());
-                return;
-            }
-            throw denial(auditId, authentication, action, request, permission.names());
-        } else if (isTranslatedToBulkAction(action)) {
-            if (request instanceof CompositeIndicesRequest == false) {
-                throw new IllegalStateException("Bulk translated actions must implement " + CompositeIndicesRequest.class.getSimpleName()
-                    + ", " + request.getClass().getSimpleName() + " doesn't");
-            }
-            // we check if the user can execute the action, without looking at indices, which will be authorized at the shard level
-            if (permission.checkIndicesAction(action)) {
-                auditTrail.accessGranted(auditId, authentication, action, request, permission.names());
-                return;
-            }
-            throw denial(auditId, authentication, action, request, permission.names());
-        } else if (TransportActionProxy.isProxyAction(action)) {
-            // we authorize proxied actions once they are "unwrapped" on the next node
-            if (TransportActionProxy.isProxyRequest(originalRequest) == false) {
-                throw new IllegalStateException("originalRequest is not a proxy request: [" + originalRequest + "] but action: ["
-                    + action + "] is a proxy action");
-            }
-            if (permission.checkIndicesAction(action)) {
-                auditTrail.accessGranted(auditId, authentication, action, request, permission.names());
-                return;
+        //if we are creating an index we need to authorize potential aliases created at the same time
+        if (IndexPrivilege.CREATE_INDEX_MATCHER.test(action)) {
+            assert request instanceof CreateIndexRequest;
+            Set<Alias> aliases = ((CreateIndexRequest) request).aliases();
+            if (aliases.isEmpty()) {
+                runRequestInterceptors(requestInfo, authzInfo, authorizationEngine, listener);
             } else {
-                // we do this here in addition to the denial below since we might run into an assertion on scroll request below if we
-                // don't have permission to read cross cluster but wrap a scroll request.
-                throw denial(auditId, authentication, action, request, permission.names());
+                final RequestInfo aliasesRequestInfo = new RequestInfo(authentication, request, IndicesAliasesAction.NAME);
+                authzEngine.authorizeIndexAction(aliasesRequestInfo, authzInfo,
+                    ril -> {
+                        resolvedIndicesAsyncSupplier.getAsync(ActionListener.wrap(resolvedIndices -> {
+                            List<String> aliasesAndIndices = new ArrayList<>(resolvedIndices.getLocal());
+                            for (Alias alias : aliases) {
+                                aliasesAndIndices.add(alias.name());
+                            }
+                            ResolvedIndices withAliases = new ResolvedIndices(aliasesAndIndices, Collections.emptyList());
+                            ril.onResponse(withAliases);
+                        }, ril::onFailure));
+                    },
+                    metaData.getAliasAndIndexLookup(),
+                    wrapPreservingContext(new AuthorizationResultListener<>(
+                        authorizationResult -> runRequestInterceptors(requestInfo, authzInfo, authorizationEngine, listener),
+                        listener::onFailure, aliasesRequestInfo, requestId, authzInfo), threadContext));
             }
+        } else if (action.equals(TransportShardBulkAction.ACTION_NAME)) {
+            // if this is performing multiple actions on the index, then check each of those actions.
+            assert request instanceof BulkShardRequest
+                : "Action " + action + " requires " + BulkShardRequest.class + " but was " + request.getClass();
+            authorizeBulkItems(requestInfo, authzInfo, authzEngine, resolvedIndicesAsyncSupplier, authorizedIndicesSupplier,
+                metaData, requestId,
+                ActionListener.wrap(ignore -> runRequestInterceptors(requestInfo, authzInfo, authorizationEngine, listener),
+                    listener::onFailure));
+        } else {
+            runRequestInterceptors(requestInfo, authzInfo, authorizationEngine, listener);
         }
+    }
 
-        // some APIs are indices requests that are not actually associated with indices. For example,
-        // search scroll request, is categorized under the indices context, but doesn't hold indices names
-        // (in this case, the security check on the indices was done on the search request that initialized
-        // the scroll. Given that scroll is implemented using a context on the node holding the shard, we
-        // piggyback on it and enhance the context with the original authentication. This serves as our method
-        // to validate the scroll id only stays with the same user!
-        if (request instanceof IndicesRequest == false && request instanceof IndicesAliasesRequest == false) {
-            //note that clear scroll shard level actions can originate from a clear scroll all, which doesn't require any
-            //indices permission as it's categorized under cluster. This is why the scroll check is performed
-            //even before checking if the user has any indices permission.
-            if (isScrollRelatedAction(action)) {
-                // if the action is a search scroll action, we first authorize that the user can execute the action for some
-                // index and if they cannot, we can fail the request early before we allow the execution of the action and in
-                // turn the shard actions
-                if (SearchScrollAction.NAME.equals(action) && permission.checkIndicesAction(action) == false) {
-                    throw denial(auditId, authentication, action, request, permission.names());
-                } else {
-                    // we store the request as a transient in the ThreadContext in case of a authorization failure at the shard
-                    // level. If authorization fails we will audit a access_denied message and will use the request to retrieve
-                    // information such as the index and the incoming address of the request
-                    auditTrail.accessGranted(auditId, authentication, action, request, permission.names());
-                    return;
-                }
-            } else {
-                assert false :
-                    "only scroll related requests are known indices api that don't support retrieving the indices they relate to";
-                throw denial(auditId, authentication, action, request, permission.names());
+    private void runRequestInterceptors(RequestInfo requestInfo, AuthorizationInfo authorizationInfo,
+                                        AuthorizationEngine authorizationEngine, ActionListener<Void> listener) {
+        if (requestInterceptors.isEmpty()) {
+            listener.onResponse(null);
+        } else {
+            Iterator<RequestInterceptor> requestInterceptorIterator = requestInterceptors.iterator();
+            final StepListener<Void> firstStepListener = new StepListener<>();
+            final RequestInterceptor first = requestInterceptorIterator.next();
+
+            StepListener<Void> prevListener = firstStepListener;
+            while (requestInterceptorIterator.hasNext()) {
+                final RequestInterceptor nextInterceptor = requestInterceptorIterator.next();
+                final StepListener<Void> current = new StepListener<>();
+                prevListener.whenComplete(v -> nextInterceptor.intercept(requestInfo, authorizationEngine, authorizationInfo, current),
+                    listener::onFailure);
+                prevListener = current;
             }
+
+            prevListener.whenComplete(v -> listener.onResponse(null), listener::onFailure);
+            first.intercept(requestInfo, authorizationEngine, authorizationInfo, firstStepListener);
         }
+    }
 
-        final boolean allowsRemoteIndices = request instanceof IndicesRequest
-            && IndicesAndAliasesResolver.allowsRemoteIndices((IndicesRequest) request);
 
-        // If this request does not allow remote indices
-        // then the user must have permission to perform this action on at least 1 local index
-        if (allowsRemoteIndices == false && permission.checkIndicesAction(action) == false) {
-            throw denial(auditId, authentication, action, request, permission.names());
-        }
+    // pkg-private for testing
+    AuthorizationEngine getRunAsAuthorizationEngine(final Authentication authentication) {
+        return getAuthorizationEngineForUser(authentication.getUser().authenticatedUser());
+    }
 
-        final MetaData metaData = clusterService.state().metaData();
-        final AuthorizedIndices authorizedIndices = new AuthorizedIndices(permission, action, metaData);
-        final ResolvedIndices resolvedIndices = resolveIndexNames(auditId, authentication, action, request, metaData,
-            authorizedIndices, permission);
-        assert !resolvedIndices.isEmpty()
-            : "every indices request needs to have its indices set thus the resolved indices must not be empty";
-
-        // If this request does reference any remote indices
-        // then the user must have permission to perform this action on at least 1 local index
-        if (resolvedIndices.getRemote().isEmpty() && permission.checkIndicesAction(action) == false) {
-            throw denial(auditId, authentication, action, request, permission.names());
-        }
+    // pkg-private for testing
+    AuthorizationEngine getAuthorizationEngine(final Authentication authentication) {
+        return getAuthorizationEngineForUser(authentication.getUser());
+    }
 
-        //all wildcard expressions have been resolved and only the security plugin could have set '-*' here.
-        //'-*' matches no indices so we allow the request to go through, which will yield an empty response
-        if (resolvedIndices.isNoIndicesPlaceholder()) {
-            putTransientIfNonExisting(AuthorizationServiceField.INDICES_PERMISSIONS_KEY, IndicesAccessControl.ALLOW_NO_INDICES);
-            auditTrail.accessGranted(auditId, authentication, action, request, permission.names());
-            return;
+    private AuthorizationEngine getAuthorizationEngineForUser(final User user) {
+        if (rbacEngine != authorizationEngine && licenseState.isAuthorizationEngineAllowed()) {
+            if (ClientReservedRealm.isReserved(user.principal(), settings) || isInternalUser(user)) {
+                return rbacEngine;
+            } else {
+                return authorizationEngine;
+            }
+        } else {
+            return rbacEngine;
         }
+    }
 
-        final Set<String> localIndices = new HashSet<>(resolvedIndices.getLocal());
-        IndicesAccessControl indicesAccessControl = permission.authorize(action, localIndices, metaData, fieldPermissionsCache);
-        if (indicesAccessControl.isGranted()) {
-            putTransientIfNonExisting(AuthorizationServiceField.INDICES_PERMISSIONS_KEY, indicesAccessControl);
+    private void authorizeSystemUser(final Authentication authentication, final String action, final String requestId,
+                                     final TransportRequest request, final ActionListener<Void> listener) {
+        if (SystemUser.isAuthorized(action)) {
+            putTransientIfNonExisting(AuthorizationServiceField.INDICES_PERMISSIONS_KEY, IndicesAccessControl.ALLOW_ALL);
+            putTransientIfNonExisting(AUTHORIZATION_INFO_KEY, SYSTEM_AUTHZ_INFO);
+            auditTrail.accessGranted(requestId, authentication, action, request, SYSTEM_AUTHZ_INFO);
+            listener.onResponse(null);
         } else {
-            throw denial(auditId, authentication, action, request, permission.names());
+            auditTrail.accessDenied(requestId, authentication, action, request, SYSTEM_AUTHZ_INFO);
+            listener.onFailure(denialException(authentication, action, null));
         }
+    }
 
-        //if we are creating an index we need to authorize potential aliases created at the same time
-        if (IndexPrivilege.CREATE_INDEX_MATCHER.test(action)) {
-            assert request instanceof CreateIndexRequest;
-            Set<Alias> aliases = ((CreateIndexRequest) request).aliases();
-            if (!aliases.isEmpty()) {
-                Set<String> aliasesAndIndices = Sets.newHashSet(localIndices);
-                for (Alias alias : aliases) {
-                    aliasesAndIndices.add(alias.name());
-                }
-                indicesAccessControl = permission.authorize("indices:admin/aliases", aliasesAndIndices, metaData, fieldPermissionsCache);
-                if (!indicesAccessControl.isGranted()) {
-                    throw denial(auditId, authentication, "indices:admin/aliases", request, permission.names());
-                }
-                // no need to re-add the indicesAccessControl in the context,
-                // because the create index call doesn't do anything FLS or DLS
+    private TransportRequest maybeUnwrapRequest(Authentication authentication, TransportRequest originalRequest, String action,
+                                                String requestId) {
+        final TransportRequest request;
+        if (originalRequest instanceof ConcreteShardRequest) {
+            request = ((ConcreteShardRequest<?>) originalRequest).getRequest();
+            assert TransportActionProxy.isProxyRequest(request) == false : "expected non-proxy request for action: " + action;
+        } else {
+            request = TransportActionProxy.unwrapRequest(originalRequest);
+            final boolean isOriginalRequestProxyRequest = TransportActionProxy.isProxyRequest(originalRequest);
+            final boolean isProxyAction = TransportActionProxy.isProxyAction(action);
+            if (isProxyAction && isOriginalRequestProxyRequest == false) {
+                IllegalStateException cause = new IllegalStateException("originalRequest is not a proxy request: [" + originalRequest +
+                    "] but action: [" + action + "] is a proxy action");
+                auditTrail.accessDenied(requestId, authentication, action, request, EmptyAuthorizationInfo.INSTANCE);
+                throw denialException(authentication, action, cause);
+            }
+            if (TransportActionProxy.isProxyRequest(originalRequest) && TransportActionProxy.isProxyAction(action) == false) {
+                IllegalStateException cause = new IllegalStateException("originalRequest is a proxy request for: [" + request +
+                    "] but action: [" + action + "] isn't");
+                auditTrail.accessDenied(requestId, authentication, action, request, EmptyAuthorizationInfo.INSTANCE);
+                throw denialException(authentication, action, cause);
             }
         }
-
-        if (action.equals(TransportShardBulkAction.ACTION_NAME)) {
-            // is this is performing multiple actions on the index, then check each of those actions.
-            assert request instanceof BulkShardRequest
-                : "Action " + action + " requires " + BulkShardRequest.class + " but was " + request.getClass();
-
-            authorizeBulkItems(auditId, authentication, (BulkShardRequest) request, permission, metaData, localIndices, authorizedIndices);
-        }
-
-        auditTrail.accessGranted(auditId, authentication, action, request, permission.names());
+        return request;
     }
 
     private boolean isInternalUser(User user) {
         return SystemUser.is(user) || XPackUser.is(user) || XPackSecurityUser.is(user);
     }
 
+    private void authorizeRunAs(final RequestInfo requestInfo, final AuthorizationInfo authzInfo,
+                                final ActionListener<AuthorizationResult> listener) {
+        final Authentication authentication = requestInfo.getAuthentication();
+        if (authentication.getLookedUpBy() == null) {
+            // this user did not really exist
+            // TODO(jaymode) find a better way to indicate lookup failed for a user and we need to fail authz
+            listener.onResponse(AuthorizationResult.deny());
+        } else {
+            final AuthorizationEngine runAsAuthzEngine = getRunAsAuthorizationEngine(authentication);
+            runAsAuthzEngine.authorizeRunAs(requestInfo, authzInfo, listener);
+        }
+    }
+
     /**
      * Performs authorization checks on the items within a {@link BulkShardRequest}.
      * This inspects the {@link BulkItemRequest items} within the request, computes
@@ -357,48 +426,99 @@ public class AuthorizationService {
      * and then checks whether that action is allowed on the targeted index. Items
      * that fail this checks are {@link BulkItemRequest#abort(String, Exception)
      * aborted}, with an
-     * {@link #denial(String, Authentication, String, TransportRequest, String[]) access
+     * {@link #denialException(Authentication, String, Exception) access
      * denied} exception. Because a shard level request is for exactly 1 index, and
      * there are a small number of possible item {@link DocWriteRequest.OpType
      * types}, the number of distinct authorization checks that need to be performed
      * is very small, but the results must be cached, to avoid adding a high
      * overhead to each bulk request.
      */
-    private void authorizeBulkItems(String auditRequestId, Authentication authentication, BulkShardRequest request, Role permission,
-                                    MetaData metaData, Set<String> indices, AuthorizedIndices authorizedIndices) {
+    private void authorizeBulkItems(RequestInfo requestInfo, AuthorizationInfo authzInfo,
+                                    AuthorizationEngine authzEngine, AsyncSupplier<ResolvedIndices> resolvedIndicesAsyncSupplier,
+                                    AsyncSupplier<List<String>> authorizedIndicesSupplier,
+                                    MetaData metaData, String requestId, ActionListener<Void> listener) {
+        final Authentication authentication = requestInfo.getAuthentication();
+        final BulkShardRequest request = (BulkShardRequest) requestInfo.getRequest();
         // Maps original-index -> expanded-index-name (expands date-math, but not aliases)
         final Map<String, String> resolvedIndexNames = new HashMap<>();
-        // Maps (resolved-index , action) -> is-granted
-        final Map<Tuple<String, String>, Boolean> indexActionAuthority = new HashMap<>();
-        for (BulkItemRequest item : request.items()) {
-            String resolvedIndex = resolvedIndexNames.computeIfAbsent(item.index(), key -> {
-                final ResolvedIndices resolvedIndices = indicesAndAliasesResolver.resolveIndicesAndAliases(item.request(), metaData,
-                    authorizedIndices);
-                if (resolvedIndices.getRemote().size() != 0) {
-                    throw illegalArgument("Bulk item should not write to remote indices, but request writes to "
-                        + String.join(",", resolvedIndices.getRemote()));
-                }
-                if (resolvedIndices.getLocal().size() != 1) {
-                    throw illegalArgument("Bulk item should write to exactly 1 index, but request writes to "
-                        + String.join(",", resolvedIndices.getLocal()));
+        // Maps action -> resolved indices set
+        final Map<String, Set<String>> actionToIndicesMap = new HashMap<>();
+
+        authorizedIndicesSupplier.getAsync(ActionListener.wrap(authorizedIndices -> {
+            resolvedIndicesAsyncSupplier.getAsync(ActionListener.wrap(overallResolvedIndices -> {
+                final Set<String> localIndices = new HashSet<>(overallResolvedIndices.getLocal());
+                for (BulkItemRequest item : request.items()) {
+                    String resolvedIndex = resolvedIndexNames.computeIfAbsent(item.index(), key -> {
+                        final ResolvedIndices resolvedIndices =
+                            indicesAndAliasesResolver.resolveIndicesAndAliases(item.request(), metaData, authorizedIndices);
+                        if (resolvedIndices.getRemote().size() != 0) {
+                            throw illegalArgument("Bulk item should not write to remote indices, but request writes to "
+                                + String.join(",", resolvedIndices.getRemote()));
+                        }
+                        if (resolvedIndices.getLocal().size() != 1) {
+                            throw illegalArgument("Bulk item should write to exactly 1 index, but request writes to "
+                                + String.join(",", resolvedIndices.getLocal()));
+                        }
+                        final String resolved = resolvedIndices.getLocal().get(0);
+                        if (localIndices.contains(resolved) == false) {
+                            throw illegalArgument("Found bulk item that writes to index " + resolved + " but the request writes to " +
+                                localIndices);
+                        }
+                        return resolved;
+                    });
+
+                    final String itemAction = getAction(item);
+                    actionToIndicesMap.compute(itemAction, (key, resolvedIndicesSet) -> {
+                        final Set<String> localSet = resolvedIndicesSet != null ? resolvedIndicesSet : new HashSet<>();
+                        localSet.add(resolvedIndex);
+                        return localSet;
+                    });
                 }
-                final String resolved = resolvedIndices.getLocal().get(0);
-                if (indices.contains(resolved) == false) {
-                    throw illegalArgument("Found bulk item that writes to index " + resolved + " but the request writes to " + indices);
-                }
-                return resolved;
-            });
-            final String itemAction = getAction(item);
-            final Tuple<String, String> indexAndAction = new Tuple<>(resolvedIndex, itemAction);
-            final boolean granted = indexActionAuthority.computeIfAbsent(indexAndAction, key -> {
-                final IndicesAccessControl itemAccessControl = permission.authorize(itemAction, Collections.singleton(resolvedIndex),
-                    metaData, fieldPermissionsCache);
-                return itemAccessControl.isGranted();
-            });
-            if (granted == false) {
-                item.abort(resolvedIndex, denial(auditRequestId, authentication, itemAction, request, permission.names()));
-            }
-        }
+
+                final ActionListener<Collection<Tuple<String, IndexAuthorizationResult>>> bulkAuthzListener =
+                    ActionListener.wrap(collection -> {
+                        final Map<String, IndicesAccessControl> actionToIndicesAccessControl = new HashMap<>();
+                        final AtomicBoolean audit = new AtomicBoolean(false);
+                        collection.forEach(tuple -> {
+                            final IndicesAccessControl existing =
+                                actionToIndicesAccessControl.putIfAbsent(tuple.v1(), tuple.v2().getIndicesAccessControl());
+                            if (existing != null) {
+                                throw new IllegalStateException("a value already exists for action " + tuple.v1());
+                            }
+                            if (tuple.v2().isAuditable()) {
+                                audit.set(true);
+                            }
+                        });
+
+                        for (BulkItemRequest item : request.items()) {
+                            final String resolvedIndex = resolvedIndexNames.get(item.index());
+                            final String itemAction = getAction(item);
+                            final IndicesAccessControl indicesAccessControl = actionToIndicesAccessControl.get(getAction(item));
+                            final IndicesAccessControl.IndexAccessControl indexAccessControl
+                                = indicesAccessControl.getIndexPermissions(resolvedIndex);
+                            if (indexAccessControl == null || indexAccessControl.isGranted() == false) {
+                                auditTrail.accessDenied(requestId, authentication, itemAction, request, authzInfo);
+                                item.abort(resolvedIndex, denialException(authentication, itemAction, null));
+                            } else if (audit.get()) {
+                                auditTrail.accessGranted(requestId, authentication, itemAction, request, authzInfo);
+                            }
+                        }
+                        listener.onResponse(null);
+                    }, listener::onFailure);
+                final ActionListener<Tuple<String, IndexAuthorizationResult>> groupedActionListener = wrapPreservingContext(
+                    new GroupedActionListener<>(bulkAuthzListener, actionToIndicesMap.size(), Collections.emptyList()), threadContext);
+
+                actionToIndicesMap.forEach((bulkItemAction, indices) -> {
+                    final RequestInfo bulkItemInfo =
+                        new RequestInfo(requestInfo.getAuthentication(), requestInfo.getRequest(), bulkItemAction);
+                    authzEngine.authorizeIndexAction(bulkItemInfo, authzInfo,
+                        ril -> ril.onResponse(new ResolvedIndices(new ArrayList<>(indices), Collections.emptyList())),
+                        metaData.getAliasAndIndexLookup(), ActionListener.wrap(indexAuthorizationResult ->
+                                groupedActionListener.onResponse(new Tuple<>(bulkItemAction, indexAuthorizationResult)),
+                            groupedActionListener::onFailure));
+                });
+                }, listener::onFailure));
+            }, listener::onFailure));
     }
 
     private IllegalArgumentException illegalArgument(String message) {
@@ -420,14 +540,9 @@ public class AuthorizationService {
         throw new IllegalArgumentException("No equivalent action for opType [" + docWriteRequest.opType() + "]");
     }
 
-    private ResolvedIndices resolveIndexNames(String auditRequestId, Authentication authentication, String action, TransportRequest request,
-                                              MetaData metaData, AuthorizedIndices authorizedIndices, Role permission) {
-        try {
-            return indicesAndAliasesResolver.resolve(request, metaData, authorizedIndices);
-        } catch (Exception e) {
-            auditTrail.accessDenied(auditRequestId, authentication, action, request, permission.names());
-            throw e;
-        }
+    private void resolveIndexNames(TransportRequest request, MetaData metaData, List<String> authorizedIndices,
+                                   ActionListener<ResolvedIndices> listener) {
+        listener.onResponse(indicesAndAliasesResolver.resolve(request, metaData, authorizedIndices));
     }
 
     private void putTransientIfNonExisting(String key, Object value) {
@@ -437,155 +552,93 @@ public class AuthorizationService {
         }
     }
 
-    public void roles(User user, Authentication authentication, ActionListener<Role> roleActionListener) {
-        // we need to special case the internal users in this method, if we apply the anonymous roles to every user including these system
-        // user accounts then we run into the chance of a deadlock because then we need to get a role that we may be trying to get as the
-        // internal user. The SystemUser is special cased as it has special privileges to execute internal actions and should never be
-        // passed into this method. The XPackUser has the Superuser role and we can simply return that
-        if (SystemUser.is(user)) {
-            throw new IllegalArgumentException("the user [" + user.principal() + "] is the system user and we should never try to get its" +
-                " roles");
+    private ElasticsearchSecurityException denialException(Authentication authentication, String action, Exception cause) {
+        final User authUser = authentication.getUser().authenticatedUser();
+        // Special case for anonymous user
+        if (isAnonymousEnabled && anonymousUser.equals(authUser)) {
+            if (anonymousAuthzExceptionEnabled == false) {
+                return authcFailureHandler.authenticationRequired(action, threadContext);
+            }
         }
-        if (XPackUser.is(user)) {
-            assert XPackUser.INSTANCE.roles().length == 1;
-            roleActionListener.onResponse(XPackUser.ROLE);
-            return;
+        // check for run as
+        if (authentication.getUser().isRunAs()) {
+            logger.debug("action [{}] is unauthorized for user [{}] run as [{}]", action, authUser.principal(),
+                    authentication.getUser().principal());
+            return authorizationError("action [{}] is unauthorized for user [{}] run as [{}]", cause, action, authUser.principal(),
+                    authentication.getUser().principal());
         }
-        if (XPackSecurityUser.is(user)) {
-            roleActionListener.onResponse(ReservedRolesStore.SUPERUSER_ROLE);
-            return;
+        logger.debug("action [{}] is unauthorized for user [{}]", action, authUser.principal());
+        return authorizationError("action [{}] is unauthorized for user [{}]", cause, action, authUser.principal());
+    }
+
+    private class AuthorizationResultListener<T extends AuthorizationResult> implements ActionListener<T> {
+
+        private final Consumer<T> responseConsumer;
+        private final Consumer<Exception> failureConsumer;
+        private final RequestInfo requestInfo;
+        private final String requestId;
+        private final AuthorizationInfo authzInfo;
+
+        private AuthorizationResultListener(Consumer<T> responseConsumer, Consumer<Exception> failureConsumer, RequestInfo requestInfo,
+                                            String requestId, AuthorizationInfo authzInfo) {
+            this.responseConsumer = responseConsumer;
+            this.failureConsumer = failureConsumer;
+            this.requestInfo = requestInfo;
+            this.requestId = requestId;
+            this.authzInfo = authzInfo;
         }
 
-        final Authentication.AuthenticationType authType = authentication.getAuthenticationType();
-        if (authType == Authentication.AuthenticationType.API_KEY) {
-            apiKeyService.getRoleForApiKey(authentication, rolesStore, roleActionListener);
-        } else {
-            Set<String> roleNames = new HashSet<>();
-            Collections.addAll(roleNames, user.roles());
-            if (isAnonymousEnabled && anonymousUser.equals(user) == false) {
-                if (anonymousUser.roles().length == 0) {
-                    throw new IllegalStateException("anonymous is only enabled when the anonymous user has roles");
+        @Override
+        public void onResponse(T result) {
+            if (result.isGranted()) {
+                if (result.isAuditable()) {
+                    auditTrail.accessGranted(requestId, requestInfo.getAuthentication(), requestInfo.getAction(), requestInfo.getRequest(),
+                        authzInfo);
+                }
+                try {
+                    responseConsumer.accept(result);
+                } catch (Exception e) {
+                    failureConsumer.accept(e);
                 }
-                Collections.addAll(roleNames, anonymousUser.roles());
-            }
-
-            if (roleNames.isEmpty()) {
-                roleActionListener.onResponse(Role.EMPTY);
-            } else if (roleNames.contains(ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR.getName())) {
-                roleActionListener.onResponse(ReservedRolesStore.SUPERUSER_ROLE);
             } else {
-                rolesStore.roles(roleNames, roleActionListener);
+                handleFailure(result.isAuditable(), null);
             }
         }
-    }
-
-    private static boolean isCompositeAction(String action) {
-        return action.equals(BulkAction.NAME) ||
-            action.equals(MultiGetAction.NAME) ||
-            action.equals(MultiTermVectorsAction.NAME) ||
-            action.equals(MultiSearchAction.NAME) ||
-            action.equals("indices:data/read/mpercolate") ||
-            action.equals("indices:data/read/msearch/template") ||
-            action.equals("indices:data/read/search/template") ||
-            action.equals("indices:data/write/reindex") ||
-            action.equals("indices:data/read/sql") ||
-            action.equals("indices:data/read/sql/translate");
-    }
-
-    private static boolean isTranslatedToBulkAction(String action) {
-        return action.equals(IndexAction.NAME) ||
-            action.equals(DeleteAction.NAME) ||
-            action.equals(INDEX_SUB_REQUEST_PRIMARY) ||
-            action.equals(INDEX_SUB_REQUEST_REPLICA) ||
-            action.equals(DELETE_SUB_REQUEST_PRIMARY) ||
-            action.equals(DELETE_SUB_REQUEST_REPLICA);
-    }
 
-    private static boolean isScrollRelatedAction(String action) {
-        return action.equals(SearchScrollAction.NAME) ||
-            action.equals(SearchTransportService.FETCH_ID_SCROLL_ACTION_NAME) ||
-            action.equals(SearchTransportService.QUERY_FETCH_SCROLL_ACTION_NAME) ||
-            action.equals(SearchTransportService.QUERY_SCROLL_ACTION_NAME) ||
-            action.equals(SearchTransportService.FREE_CONTEXT_SCROLL_ACTION_NAME) ||
-            action.equals(ClearScrollAction.NAME) ||
-            action.equals("indices:data/read/sql/close_cursor") ||
-            action.equals(SearchTransportService.CLEAR_SCROLL_CONTEXTS_ACTION_NAME);
-    }
+        @Override
+        public void onFailure(Exception e) {
+            handleFailure(true, e);
+        }
 
-    static boolean checkSameUserPermissions(String action, TransportRequest request, Authentication authentication) {
-        final boolean actionAllowed = SAME_USER_PRIVILEGE.test(action);
-        if (actionAllowed) {
-            if (request instanceof UserRequest == false) {
-                assert false : "right now only a user request should be allowed";
-                return false;
-            }
-            UserRequest userRequest = (UserRequest) request;
-            String[] usernames = userRequest.usernames();
-            if (usernames == null || usernames.length != 1 || usernames[0] == null) {
-                assert false : "this role should only be used for actions to apply to a single user";
-                return false;
-            }
-            final String username = usernames[0];
-            final boolean sameUsername = authentication.getUser().principal().equals(username);
-            if (sameUsername && ChangePasswordAction.NAME.equals(action)) {
-                return checkChangePasswordAction(authentication);
+        private void handleFailure(boolean audit, @Nullable Exception e) {
+            if (audit) {
+                auditTrail.accessDenied(requestId, requestInfo.getAuthentication(), requestInfo.getAction(), requestInfo.getRequest(),
+                    authzInfo);
             }
-
-            assert AuthenticateAction.NAME.equals(action) || HasPrivilegesAction.NAME.equals(action)
-                || GetUserPrivilegesAction.NAME.equals(action) || sameUsername == false
-                : "Action '" + action + "' should not be possible when sameUsername=" + sameUsername;
-            return sameUsername;
+            failureConsumer.accept(denialException(requestInfo.getAuthentication(), requestInfo.getAction(), e));
         }
-        return false;
     }
 
-    private static boolean checkChangePasswordAction(Authentication authentication) {
-        // we need to verify that this user was authenticated by or looked up by a realm type that support password changes
-        // otherwise we open ourselves up to issues where a user in a different realm could be created with the same username
-        // and do malicious things
-        final boolean isRunAs = authentication.getUser().isRunAs();
-        final String realmType;
-        if (isRunAs) {
-            realmType = authentication.getLookedUpBy().getType();
-        } else {
-            realmType = authentication.getAuthenticatedBy().getType();
-        }
-
-        assert realmType != null;
-        // ensure the user was authenticated by a realm that we can change a password for. The native realm is an internal realm and
-        // right now only one can exist in the realm configuration - if this changes we should update this check
-        return ReservedRealm.TYPE.equals(realmType) || NativeRealmSettings.TYPE.equals(realmType);
-    }
+    private static class CachingAsyncSupplier<V> implements AsyncSupplier<V> {
 
-    ElasticsearchSecurityException denial(String auditRequestId, Authentication authentication, String action, TransportRequest request,
-                                          String[] roleNames) {
-        auditTrail.accessDenied(auditRequestId, authentication, action, request, roleNames);
-        return denialException(authentication, action);
-    }
+        private final AsyncSupplier<V> asyncSupplier;
+        private V value = null;
 
-    private ElasticsearchSecurityException denyRunAs(String auditRequestId, Authentication authentication, String action,
-                                                     TransportRequest request, String[] roleNames) {
-        auditTrail.runAsDenied(auditRequestId, authentication, action, request, roleNames);
-        return denialException(authentication, action);
-    }
+        private CachingAsyncSupplier(AsyncSupplier<V> supplier) {
+            this.asyncSupplier = supplier;
+        }
 
-    private ElasticsearchSecurityException denialException(Authentication authentication, String action) {
-        final User authUser = authentication.getUser().authenticatedUser();
-        // Special case for anonymous user
-        if (isAnonymousEnabled && anonymousUser.equals(authUser)) {
-            if (anonymousAuthzExceptionEnabled == false) {
-                throw authcFailureHandler.authenticationRequired(action, threadContext);
+        @Override
+        public synchronized void getAsync(ActionListener<V> listener) {
+            if (value == null) {
+                asyncSupplier.getAsync(ActionListener.wrap(loaded -> {
+                    value = loaded;
+                    listener.onResponse(value);
+                }, listener::onFailure));
+            } else {
+                listener.onResponse(value);
             }
         }
-        // check for run as
-        if (authentication.getUser().isRunAs()) {
-            logger.debug("action [{}] is unauthorized for user [{}] run as [{}]", action, authUser.principal(),
-                authentication.getUser().principal());
-            return authorizationError("action [{}] is unauthorized for user [{}] run as [{}]", action, authUser.principal(),
-                authentication.getUser().principal());
-        }
-        logger.debug("action [{}] is unauthorized for user [{}]", action, authUser.principal());
-        return authorizationError("action [{}] is unauthorized for user [{}]", action, authUser.principal());
     }
 
     public static void addSettings(List<Setting<?>> settings) {

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java

@@ -6,20 +6,15 @@
 package org.elasticsearch.xpack.security.authz;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.common.util.concurrent.CountDown;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.security.SecurityContext;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationField;
-import org.elasticsearch.xpack.core.security.authz.permission.Role;
 import org.elasticsearch.xpack.core.security.support.Automatons;
-import org.elasticsearch.xpack.core.security.user.SystemUser;
 import org.elasticsearch.xpack.core.security.user.XPackSecurityUser;
 import org.elasticsearch.xpack.core.security.user.XPackUser;
 
-import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.function.Predicate;
 
@@ -126,61 +121,4 @@ public final class AuthorizationUtils {
     private static boolean isInternalAction(String action) {
         return INTERNAL_PREDICATE.test(action);
     }
-
-    /**
-     * A base class to authorize authorize a given {@link Authentication} against it's users or run-as users roles.
-     * This class fetches the roles for the users asynchronously and then authenticates the in the callback.
-     */
-    public static class AsyncAuthorizer {
-
-        private final ActionListener<Void> listener;
-        private final BiConsumer<Role, Role> consumer;
-        private final Authentication authentication;
-        private volatile Role userRoles;
-        private volatile Role runAsRoles;
-        private CountDown countDown = new CountDown(2); // we expect only two responses!!
-
-        public AsyncAuthorizer(Authentication authentication, ActionListener<Void> listener, BiConsumer<Role, Role> consumer) {
-            this.consumer = consumer;
-            this.listener = listener;
-            this.authentication = authentication;
-        }
-
-        public void authorize(AuthorizationService service) {
-            if (SystemUser.is(authentication.getUser().authenticatedUser())) {
-                assert authentication.getUser().isRunAs() == false;
-                setUserRoles(null); // we can inform the listener immediately - nothing to fetch for us on system user
-                setRunAsRoles(null);
-            } else {
-                service.roles(authentication.getUser().authenticatedUser(), authentication,
-                    ActionListener.wrap(this::setUserRoles, listener::onFailure));
-                if (authentication.getUser().isRunAs()) {
-                    service.roles(authentication.getUser(), authentication, ActionListener.wrap(this::setRunAsRoles, listener::onFailure));
-                } else {
-                    setRunAsRoles(null);
-                }
-            }
-        }
-
-        private void setUserRoles(Role roles) {
-            this.userRoles = roles;
-            maybeRun();
-        }
-
-        private void setRunAsRoles(Role roles) {
-            this.runAsRoles = roles;
-            maybeRun();
-        }
-
-        private void maybeRun() {
-            if (countDown.countDown()) {
-                try {
-                    consumer.accept(userRoles, runAsRoles);
-                } catch (Exception e) {
-                    listener.onFailure(e);
-                }
-            }
-        }
-
-    }
 }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizedIndices.java

@@ -1,55 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.security.authz;
-
-import org.elasticsearch.cluster.metadata.AliasOrIndex;
-import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.xpack.core.security.authz.permission.Role;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.function.Predicate;
-
-/**
- * Abstraction used to make sure that we lazily load authorized indices only when requested and only maximum once per request. Also
- * makes sure that authorized indices don't get updated throughout the same request for the same user.
- */
-class AuthorizedIndices {
-    private final String action;
-    private final MetaData metaData;
-    private final Role userRoles;
-    private List<String> authorizedIndices;
-
-    AuthorizedIndices(Role userRoles, String action, MetaData metaData) {
-        this.userRoles = userRoles;
-        this.action = action;
-        this.metaData = metaData;
-    }
-
-    List<String> get() {
-        if (authorizedIndices == null) {
-            authorizedIndices = load();
-        }
-        return authorizedIndices;
-    }
-
-    private List<String> load() {
-        Predicate<String> predicate = userRoles.allowedIndicesMatcher(action);
-
-        List<String> indicesAndAliases = new ArrayList<>();
-        // TODO: can this be done smarter? I think there are usually more indices/aliases in the cluster then indices defined a roles?
-        for (Map.Entry<String, AliasOrIndex> entry : metaData.getAliasAndIndexLookup().entrySet()) {
-            String aliasOrIndex = entry.getKey();
-            if (predicate.test(aliasOrIndex)) {
-                indicesAndAliases.add(aliasOrIndex);
-            }
-        }
-
-        return Collections.unmodifiableList(indicesAndAliases);
-    }
-}

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java

@@ -28,7 +28,7 @@ import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.protocol.xpack.graph.GraphExploreRequest;
 import org.elasticsearch.transport.RemoteClusterAware;
 import org.elasticsearch.transport.TransportRequest;
-import org.elasticsearch.xpack.core.security.authz.IndicesAndAliasesResolverField;
+import org.elasticsearch.xpack.core.security.authz.ResolvedIndices;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -87,7 +87,7 @@ class IndicesAndAliasesResolver {
      * Otherwise, <em>N</em> will be added to the <em>local</em> index list.
      */
 
-    ResolvedIndices resolve(TransportRequest request, MetaData metaData, AuthorizedIndices authorizedIndices) {
+    ResolvedIndices resolve(TransportRequest request, MetaData metaData, List<String> authorizedIndices) {
         if (request instanceof IndicesAliasesRequest) {
             ResolvedIndices.Builder resolvedIndicesBuilder = new ResolvedIndices.Builder();
             IndicesAliasesRequest indicesAliasesRequest = (IndicesAliasesRequest) request;
@@ -107,7 +107,7 @@ class IndicesAndAliasesResolver {
     }
 
 
-    ResolvedIndices resolveIndicesAndAliases(IndicesRequest indicesRequest, MetaData metaData, AuthorizedIndices authorizedIndices) {
+    ResolvedIndices resolveIndicesAndAliases(IndicesRequest indicesRequest, MetaData metaData, List<String> authorizedIndices) {
         final ResolvedIndices.Builder resolvedIndicesBuilder = new ResolvedIndices.Builder();
         boolean indicesReplacedWithNoIndices = false;
         if (indicesRequest instanceof PutMappingRequest && ((PutMappingRequest) indicesRequest).getConcreteIndex() != null) {
@@ -134,7 +134,7 @@ class IndicesAndAliasesResolver {
             // check for all and return list of authorized indices
             if (IndexNameExpressionResolver.isAllIndices(indicesList(indicesRequest.indices()))) {
                 if (replaceWildcards) {
-                    for (String authorizedIndex : authorizedIndices.get()) {
+                    for (String authorizedIndex : authorizedIndices) {
                         if (isIndexVisible(authorizedIndex, indicesOptions, metaData)) {
                             resolvedIndicesBuilder.addLocal(authorizedIndex);
                         }
@@ -150,11 +150,11 @@ class IndicesAndAliasesResolver {
                     split = new ResolvedIndices(Arrays.asList(indicesRequest.indices()), Collections.emptyList());
                 }
                 List<String> replaced = replaceWildcardsWithAuthorizedIndices(split.getLocal(), indicesOptions, metaData,
-                        authorizedIndices.get(), replaceWildcards);
+                        authorizedIndices, replaceWildcards);
                 if (indicesOptions.ignoreUnavailable()) {
                     //out of all the explicit names (expanded from wildcards and original ones that were left untouched)
                     //remove all the ones that the current user is not authorized for and ignore them
-                    replaced = replaced.stream().filter(authorizedIndices.get()::contains).collect(Collectors.toList());
+                    replaced = replaced.stream().filter(authorizedIndices::contains).collect(Collectors.toList());
                 }
                 resolvedIndicesBuilder.addLocal(replaced);
                 resolvedIndicesBuilder.addRemote(split.getRemote());
@@ -195,7 +195,7 @@ class IndicesAndAliasesResolver {
             AliasesRequest aliasesRequest = (AliasesRequest) indicesRequest;
             if (aliasesRequest.expandAliasesWildcards()) {
                 List<String> aliases = replaceWildcardsWithAuthorizedAliases(aliasesRequest.aliases(),
-                        loadAuthorizedAliases(authorizedIndices.get(), metaData));
+                        loadAuthorizedAliases(authorizedIndices, metaData));
                 aliasesRequest.replaceAliases(aliases.toArray(new String[aliases.size()]));
             }
             if (indicesReplacedWithNoIndices) {
@@ -226,9 +226,8 @@ class IndicesAndAliasesResolver {
      * request's concrete index is not in the list of authorized indices, then we need to look to
      * see if this can be authorized against an alias
      */
-    static String getPutMappingIndexOrAlias(PutMappingRequest request, AuthorizedIndices authorizedIndices, MetaData metaData) {
+    static String getPutMappingIndexOrAlias(PutMappingRequest request, List<String> authorizedIndicesList, MetaData metaData) {
         final String concreteIndexName = request.getConcreteIndex().getName();
-        final List<String> authorizedIndicesList = authorizedIndices.get();
 
         // validate that the concrete index exists, otherwise there is no remapping that we could do
         final AliasOrIndex aliasOrIndex = metaData.getAliasAndIndexLookup().get(concreteIndexName);
@@ -457,100 +456,4 @@ class IndicesAndAliasesResolver {
         }
     }
 
-    /**
-     * Stores a collection of index names separated into "local" and "remote".
-     * This allows the resolution and categorization to take place exactly once per-request.
-     */
-    public static class ResolvedIndices {
-        private final List<String> local;
-        private final List<String> remote;
-
-        ResolvedIndices(List<String> local, List<String> remote) {
-            this.local = Collections.unmodifiableList(local);
-            this.remote = Collections.unmodifiableList(remote);
-        }
-
-        /**
-         * Returns the collection of index names that have been stored as "local" indices.
-         * This is a <code>List</code> because order may be important. For example <code>[ "a*" , "-a1" ]</code> is interpreted differently
-         * to <code>[ "-a1", "a*" ]</code>. As a consequence, this list <em>may contain duplicates</em>.
-         */
-        public List<String> getLocal() {
-            return local;
-        }
-
-        /**
-         * Returns the collection of index names that have been stored as "remote" indices.
-         */
-        public List<String> getRemote() {
-            return remote;
-        }
-
-        /**
-         * @return <code>true</code> if both the {@link #getLocal() local} and {@link #getRemote() remote} index lists are empty.
-         */
-        public boolean isEmpty() {
-            return local.isEmpty() && remote.isEmpty();
-        }
-
-        /**
-         * @return <code>true</code> if the {@link #getRemote() remote} index lists is empty, and the local index list contains the
-         * {@link IndicesAndAliasesResolverField#NO_INDEX_PLACEHOLDER no-index-placeholder} and nothing else.
-         */
-        public boolean isNoIndicesPlaceholder() {
-            return remote.isEmpty() && local.size() == 1 && local.contains(NO_INDEX_PLACEHOLDER);
-        }
-
-        private String[] toArray() {
-            final String[] array = new String[local.size() + remote.size()];
-            int i = 0;
-            for (String index : local) {
-                array[i++] = index;
-            }
-            for (String index : remote) {
-                array[i++] = index;
-            }
-            return array;
-        }
-
-        /**
-         * Builder class for ResolvedIndices that allows for the building of a list of indices
-         * without the need to construct new objects and merging them together
-         */
-        private static class Builder {
-
-            private final List<String> local = new ArrayList<>();
-            private final List<String> remote = new ArrayList<>();
-
-            /** add a local index name */
-            private void addLocal(String index) {
-                local.add(index);
-            }
-
-            /** adds the array of local index names */
-            private void addLocal(String[] indices) {
-                local.addAll(Arrays.asList(indices));
-            }
-
-            /** adds the list of local index names */
-            private void addLocal(List<String> indices) {
-                local.addAll(indices);
-            }
-
-            /** adds the list of remote index names */
-            private void addRemote(List<String> indices) {
-                remote.addAll(indices);
-            }
-
-            /** @return <code>true</code> if both the local and remote index lists are empty. */
-            private boolean isEmpty() {
-                return local.isEmpty() && remote.isEmpty();
-            }
-
-            /** @return a immutable ResolvedIndices instance with the local and remote index lists */
-            private ResolvedIndices build() {
-                return new ResolvedIndices(local, remote);
-            }
-        }
-    }
 }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java

@@ -0,0 +1,552 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security.authz;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.apache.lucene.util.automaton.Automaton;
+import org.apache.lucene.util.automaton.Operations;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.CompositeIndicesRequest;
+import org.elasticsearch.action.IndicesRequest;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
+import org.elasticsearch.action.bulk.BulkAction;
+import org.elasticsearch.action.bulk.BulkShardRequest;
+import org.elasticsearch.action.delete.DeleteAction;
+import org.elasticsearch.action.get.MultiGetAction;
+import org.elasticsearch.action.index.IndexAction;
+import org.elasticsearch.action.search.ClearScrollAction;
+import org.elasticsearch.action.search.MultiSearchAction;
+import org.elasticsearch.action.search.SearchScrollAction;
+import org.elasticsearch.action.search.SearchTransportService;
+import org.elasticsearch.action.termvectors.MultiTermVectorsAction;
+import org.elasticsearch.cluster.metadata.AliasOrIndex;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.transport.TransportActionProxy;
+import org.elasticsearch.transport.TransportRequest;
+import org.elasticsearch.xpack.core.security.action.user.AuthenticateAction;
+import org.elasticsearch.xpack.core.security.action.user.ChangePasswordAction;
+import org.elasticsearch.xpack.core.security.action.user.GetUserPrivilegesAction;
+import org.elasticsearch.xpack.core.security.action.user.GetUserPrivilegesRequest;
+import org.elasticsearch.xpack.core.security.action.user.GetUserPrivilegesResponse;
+import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesAction;
+import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesRequest;
+import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesResponse;
+import org.elasticsearch.xpack.core.security.action.user.UserRequest;
+import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authc.esnative.NativeRealmSettings;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine;
+import org.elasticsearch.xpack.core.security.authz.ResolvedIndices;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
+import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsCache;
+import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsDefinition;
+import org.elasticsearch.xpack.core.security.authz.permission.IndicesPermission;
+import org.elasticsearch.xpack.core.security.authz.permission.ResourcePrivileges;
+import org.elasticsearch.xpack.core.security.authz.permission.ResourcePrivilegesMap;
+import org.elasticsearch.xpack.core.security.authz.permission.Role;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
+import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilege;
+import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivilege;
+import org.elasticsearch.xpack.core.security.authz.privilege.Privilege;
+import org.elasticsearch.xpack.core.security.support.Automatons;
+import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm;
+import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.function.Predicate;
+
+import static org.elasticsearch.common.Strings.arrayToCommaDelimitedString;
+import static org.elasticsearch.xpack.security.action.user.TransportHasPrivilegesAction.getApplicationNames;
+import static org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail.PRINCIPAL_ROLES_FIELD_NAME;
+
+public class RBACEngine implements AuthorizationEngine {
+
+    private static final Predicate<String> SAME_USER_PRIVILEGE = Automatons.predicate(
+        ChangePasswordAction.NAME, AuthenticateAction.NAME, HasPrivilegesAction.NAME, GetUserPrivilegesAction.NAME);
+    private static final String INDEX_SUB_REQUEST_PRIMARY = IndexAction.NAME + "[p]";
+    private static final String INDEX_SUB_REQUEST_REPLICA = IndexAction.NAME + "[r]";
+    private static final String DELETE_SUB_REQUEST_PRIMARY = DeleteAction.NAME + "[p]";
+    private static final String DELETE_SUB_REQUEST_REPLICA = DeleteAction.NAME + "[r]";
+
+    private static final Logger logger = LogManager.getLogger(RBACEngine.class);
+
+    private final CompositeRolesStore rolesStore;
+    private final FieldPermissionsCache fieldPermissionsCache;
+
+    public RBACEngine(Settings settings, CompositeRolesStore rolesStore) {
+        this.rolesStore = rolesStore;
+        this.fieldPermissionsCache = new FieldPermissionsCache(settings);
+    }
+
+    @Override
+    public void resolveAuthorizationInfo(RequestInfo requestInfo, ActionListener<AuthorizationInfo> listener) {
+        final Authentication authentication = requestInfo.getAuthentication();
+        getRoles(authentication.getUser(), authentication, ActionListener.wrap(role -> {
+            if (authentication.getUser().isRunAs()) {
+                getRoles(authentication.getUser().authenticatedUser(), authentication, ActionListener.wrap(
+                    authenticatedUserRole -> listener.onResponse(new RBACAuthorizationInfo(role, authenticatedUserRole)),
+                    listener::onFailure));
+            } else {
+                listener.onResponse(new RBACAuthorizationInfo(role, role));
+            }
+        }, listener::onFailure));
+    }
+
+    private void getRoles(User user, Authentication authentication, ActionListener<Role> listener) {
+        rolesStore.getRoles(user, authentication, listener);
+    }
+
+    @Override
+    public void authorizeRunAs(RequestInfo requestInfo, AuthorizationInfo authorizationInfo, ActionListener<AuthorizationResult> listener) {
+        if (authorizationInfo instanceof RBACAuthorizationInfo) {
+            final Role role = ((RBACAuthorizationInfo) authorizationInfo).getAuthenticatedUserAuthorizationInfo().getRole();
+            listener.onResponse(
+                new AuthorizationResult(role.checkRunAs(requestInfo.getAuthentication().getUser().principal())));
+        } else {
+            listener.onFailure(new IllegalArgumentException("unsupported authorization info:" +
+                authorizationInfo.getClass().getSimpleName()));
+        }
+    }
+
+    @Override
+    public void authorizeClusterAction(RequestInfo requestInfo, AuthorizationInfo authorizationInfo,
+                                       ActionListener<AuthorizationResult> listener) {
+        if (authorizationInfo instanceof RBACAuthorizationInfo) {
+            final Role role = ((RBACAuthorizationInfo) authorizationInfo).getRole();
+            if (role.checkClusterAction(requestInfo.getAction(), requestInfo.getRequest())) {
+                listener.onResponse(AuthorizationResult.granted());
+            } else if (checkSameUserPermissions(requestInfo.getAction(), requestInfo.getRequest(), requestInfo.getAuthentication())) {
+                listener.onResponse(AuthorizationResult.granted());
+            } else {
+                listener.onResponse(AuthorizationResult.deny());
+            }
+        } else {
+            listener.onFailure(new IllegalArgumentException("unsupported authorization info:" +
+                authorizationInfo.getClass().getSimpleName()));
+        }
+    }
+
+    // pkg private for testing
+    boolean checkSameUserPermissions(String action, TransportRequest request, Authentication authentication) {
+        final boolean actionAllowed = SAME_USER_PRIVILEGE.test(action);
+        if (actionAllowed) {
+            if (request instanceof UserRequest == false) {
+                assert false : "right now only a user request should be allowed";
+                return false;
+            }
+            UserRequest userRequest = (UserRequest) request;
+            String[] usernames = userRequest.usernames();
+            if (usernames == null || usernames.length != 1 || usernames[0] == null) {
+                assert false : "this role should only be used for actions to apply to a single user";
+                return false;
+            }
+            final String username = usernames[0];
+            final boolean sameUsername = authentication.getUser().principal().equals(username);
+            if (sameUsername && ChangePasswordAction.NAME.equals(action)) {
+                return checkChangePasswordAction(authentication);
+            }
+
+            assert AuthenticateAction.NAME.equals(action) || HasPrivilegesAction.NAME.equals(action)
+                || GetUserPrivilegesAction.NAME.equals(action) || sameUsername == false
+                : "Action '" + action + "' should not be possible when sameUsername=" + sameUsername;
+            return sameUsername;
+        }
+        return false;
+    }
+
+    private static boolean shouldAuthorizeIndexActionNameOnly(String action, TransportRequest request) {
+        switch (action) {
+            case BulkAction.NAME:
+            case IndexAction.NAME:
+            case DeleteAction.NAME:
+            case INDEX_SUB_REQUEST_PRIMARY:
+            case INDEX_SUB_REQUEST_REPLICA:
+            case DELETE_SUB_REQUEST_PRIMARY:
+            case DELETE_SUB_REQUEST_REPLICA:
+            case MultiGetAction.NAME:
+            case MultiTermVectorsAction.NAME:
+            case MultiSearchAction.NAME:
+            case "indices:data/read/mpercolate":
+            case "indices:data/read/msearch/template":
+            case "indices:data/read/search/template":
+            case "indices:data/write/reindex":
+            case "indices:data/read/sql":
+            case "indices:data/read/sql/translate":
+                if (request instanceof BulkShardRequest) {
+                    return false;
+                }
+                if (request instanceof CompositeIndicesRequest == false) {
+                    throw new IllegalStateException("Composite and bulk actions must implement " +
+                        CompositeIndicesRequest.class.getSimpleName() + ", " + request.getClass().getSimpleName() + " doesn't. Action " +
+                        action);
+                }
+                return true;
+            default:
+                return false;
+        }
+    }
+
+    @Override
+    public void authorizeIndexAction(RequestInfo requestInfo, AuthorizationInfo authorizationInfo,
+                                     AsyncSupplier<ResolvedIndices> indicesAsyncSupplier,
+                                     Map<String, AliasOrIndex> aliasOrIndexLookup,
+                                     ActionListener<IndexAuthorizationResult> listener) {
+        final String action = requestInfo.getAction();
+        final TransportRequest request = requestInfo.getRequest();
+        final Authentication authentication = requestInfo.getAuthentication();
+        if (TransportActionProxy.isProxyAction(action) || shouldAuthorizeIndexActionNameOnly(action, request)) {
+            // we've already validated that the request is a proxy request so we can skip that but we still
+            // need to validate that the action is allowed and then move on
+            authorizeIndexActionName(action, authorizationInfo, null, listener);
+        } else if (request instanceof IndicesRequest == false && request instanceof IndicesAliasesRequest == false) {
+            // scroll is special
+            // some APIs are indices requests that are not actually associated with indices. For example,
+            // search scroll request, is categorized under the indices context, but doesn't hold indices names
+            // (in this case, the security check on the indices was done on the search request that initialized
+            // the scroll. Given that scroll is implemented using a context on the node holding the shard, we
+            // piggyback on it and enhance the context with the original authentication. This serves as our method
+            // to validate the scroll id only stays with the same user!
+            // note that clear scroll shard level actions can originate from a clear scroll all, which doesn't require any
+            // indices permission as it's categorized under cluster. This is why the scroll check is performed
+            // even before checking if the user has any indices permission.
+            if (isScrollRelatedAction(action)) {
+                // if the action is a search scroll action, we first authorize that the user can execute the action for some
+                // index and if they cannot, we can fail the request early before we allow the execution of the action and in
+                // turn the shard actions
+                if (SearchScrollAction.NAME.equals(action)) {
+                    authorizeIndexActionName(action, authorizationInfo, null, listener);
+                } else {
+                    // we store the request as a transient in the ThreadContext in case of a authorization failure at the shard
+                    // level. If authorization fails we will audit a access_denied message and will use the request to retrieve
+                    // information such as the index and the incoming address of the request
+                    listener.onResponse(new IndexAuthorizationResult(true, IndicesAccessControl.ALLOW_NO_INDICES));
+                }
+            } else {
+                assert false :
+                    "only scroll related requests are known indices api that don't support retrieving the indices they relate to";
+                listener.onFailure(new IllegalStateException("only scroll related requests are known indices api that don't support " +
+                    "retrieving the indices they relate to"));
+            }
+        } else if (request instanceof IndicesRequest &&
+            IndicesAndAliasesResolver.allowsRemoteIndices((IndicesRequest) request)) {
+            // remote indices are allowed
+            indicesAsyncSupplier.getAsync(ActionListener.wrap(resolvedIndices -> {
+                assert !resolvedIndices.isEmpty()
+                    : "every indices request needs to have its indices set thus the resolved indices must not be empty";
+                //all wildcard expressions have been resolved and only the security plugin could have set '-*' here.
+                //'-*' matches no indices so we allow the request to go through, which will yield an empty response
+                if (resolvedIndices.isNoIndicesPlaceholder()) {
+                    // check action name
+                    authorizeIndexActionName(action, authorizationInfo, IndicesAccessControl.ALLOW_NO_INDICES, listener);
+                } else {
+                    buildIndicesAccessControl(authentication, action, authorizationInfo,
+                        Sets.newHashSet(resolvedIndices.getLocal()), aliasOrIndexLookup, listener);
+                }
+            }, listener::onFailure));
+        } else {
+            authorizeIndexActionName(action, authorizationInfo, IndicesAccessControl.ALLOW_NO_INDICES,
+                ActionListener.wrap(indexAuthorizationResult -> {
+                    if (indexAuthorizationResult.isGranted()) {
+                        indicesAsyncSupplier.getAsync(ActionListener.wrap(resolvedIndices -> {
+                            assert !resolvedIndices.isEmpty()
+                                : "every indices request needs to have its indices set thus the resolved indices must not be empty";
+                            //all wildcard expressions have been resolved and only the security plugin could have set '-*' here.
+                            //'-*' matches no indices so we allow the request to go through, which will yield an empty response
+                            if (resolvedIndices.isNoIndicesPlaceholder()) {
+                                listener.onResponse(new IndexAuthorizationResult(true, IndicesAccessControl.ALLOW_NO_INDICES));
+                            } else {
+                                buildIndicesAccessControl(authentication, action, authorizationInfo,
+                                    Sets.newHashSet(resolvedIndices.getLocal()), aliasOrIndexLookup, listener);
+                            }
+                        }, listener::onFailure));
+                    } else {
+                        listener.onResponse(indexAuthorizationResult);
+                    }
+                }, listener::onFailure));
+        }
+    }
+
+    private void authorizeIndexActionName(String action, AuthorizationInfo authorizationInfo, IndicesAccessControl grantedValue,
+                                          ActionListener<IndexAuthorizationResult> listener) {
+        if (authorizationInfo instanceof RBACAuthorizationInfo) {
+            final Role role = ((RBACAuthorizationInfo) authorizationInfo).getRole();
+            if (role.checkIndicesAction(action)) {
+                listener.onResponse(new IndexAuthorizationResult(true, grantedValue));
+            } else {
+                listener.onResponse(new IndexAuthorizationResult(true, IndicesAccessControl.DENIED));
+            }
+        } else {
+            listener.onFailure(new IllegalArgumentException("unsupported authorization info:" +
+                authorizationInfo.getClass().getSimpleName()));
+        }
+    }
+
+    @Override
+    public void loadAuthorizedIndices(RequestInfo requestInfo, AuthorizationInfo authorizationInfo,
+                                      Map<String, AliasOrIndex> aliasOrIndexLookup, ActionListener<List<String>> listener) {
+        if (authorizationInfo instanceof RBACAuthorizationInfo) {
+            final Role role = ((RBACAuthorizationInfo) authorizationInfo).getRole();
+            listener.onResponse(resolveAuthorizedIndicesFromRole(role, requestInfo.getAction(), aliasOrIndexLookup));
+        } else {
+            listener.onFailure(
+                new IllegalArgumentException("unsupported authorization info:" + authorizationInfo.getClass().getSimpleName()));
+        }
+    }
+
+    @Override
+    public void validateIndexPermissionsAreSubset(RequestInfo requestInfo, AuthorizationInfo authorizationInfo,
+                                                  Map<String, List<String>> indexNameToNewNames,
+                                                  ActionListener<AuthorizationResult> listener) {
+        if (authorizationInfo instanceof RBACAuthorizationInfo) {
+            final Role role = ((RBACAuthorizationInfo) authorizationInfo).getRole();
+            Map<String, Automaton> permissionMap = new HashMap<>();
+            for (Entry<String, List<String>> entry : indexNameToNewNames.entrySet()) {
+                Automaton existingPermissions = permissionMap.computeIfAbsent(entry.getKey(), role::allowedActionsMatcher);
+                for (String alias : entry.getValue()) {
+                    Automaton newNamePermissions = permissionMap.computeIfAbsent(alias, role::allowedActionsMatcher);
+                    if (Operations.subsetOf(newNamePermissions, existingPermissions) == false) {
+                        listener.onResponse(AuthorizationResult.deny());
+                        return;
+                    }
+                }
+            }
+            listener.onResponse(AuthorizationResult.granted());
+        } else {
+            listener.onFailure(
+                new IllegalArgumentException("unsupported authorization info:" + authorizationInfo.getClass().getSimpleName()));
+        }
+    }
+
+    @Override
+    public void checkPrivileges(Authentication authentication, AuthorizationInfo authorizationInfo,
+                                HasPrivilegesRequest request,
+                                Collection<ApplicationPrivilegeDescriptor> applicationPrivileges,
+                                ActionListener<HasPrivilegesResponse> listener) {
+        if (authorizationInfo instanceof RBACAuthorizationInfo == false) {
+            listener.onFailure(
+                new IllegalArgumentException("unsupported authorization info:" + authorizationInfo.getClass().getSimpleName()));
+            return;
+        }
+        final Role userRole = ((RBACAuthorizationInfo) authorizationInfo).getRole();
+        logger.trace(() -> new ParameterizedMessage("Check whether role [{}] has privileges cluster=[{}] index=[{}] application=[{}]",
+            Strings.arrayToCommaDelimitedString(userRole.names()),
+            Strings.arrayToCommaDelimitedString(request.clusterPrivileges()),
+            Strings.arrayToCommaDelimitedString(request.indexPrivileges()),
+            Strings.arrayToCommaDelimitedString(request.applicationPrivileges())
+        ));
+
+        Map<String, Boolean> cluster = new HashMap<>();
+        for (String checkAction : request.clusterPrivileges()) {
+            final ClusterPrivilege checkPrivilege = ClusterPrivilege.get(Collections.singleton(checkAction));
+            cluster.put(checkAction, userRole.grants(checkPrivilege));
+        }
+        boolean allMatch = cluster.values().stream().allMatch(Boolean::booleanValue);
+        ResourcePrivilegesMap.Builder combineIndicesResourcePrivileges = ResourcePrivilegesMap.builder();
+        for (RoleDescriptor.IndicesPrivileges check : request.indexPrivileges()) {
+            ResourcePrivilegesMap resourcePrivileges = userRole.checkIndicesPrivileges(Sets.newHashSet(check.getIndices()),
+                check.allowRestrictedIndices(), Sets.newHashSet(check.getPrivileges()));
+            allMatch = allMatch && resourcePrivileges.allAllowed();
+            combineIndicesResourcePrivileges.addResourcePrivilegesMap(resourcePrivileges);
+        }
+        ResourcePrivilegesMap allIndices = combineIndicesResourcePrivileges.build();
+        allMatch = allMatch && allIndices.allAllowed();
+
+        final Map<String, Collection<ResourcePrivileges>> privilegesByApplication = new HashMap<>();
+        for (String applicationName : getApplicationNames(request)) {
+            logger.debug("Checking privileges for application {}", applicationName);
+            ResourcePrivilegesMap.Builder builder = ResourcePrivilegesMap.builder();
+            for (RoleDescriptor.ApplicationResourcePrivileges p : request.applicationPrivileges()) {
+                if (applicationName.equals(p.getApplication())) {
+                    ResourcePrivilegesMap appPrivsByResourceMap = userRole.checkApplicationResourcePrivileges(applicationName,
+                        Sets.newHashSet(p.getResources()), Sets.newHashSet(p.getPrivileges()), applicationPrivileges);
+                    builder.addResourcePrivilegesMap(appPrivsByResourceMap);
+                }
+            }
+            ResourcePrivilegesMap resourcePrivsForApplication = builder.build();
+            allMatch = allMatch && resourcePrivsForApplication.allAllowed();
+            privilegesByApplication.put(applicationName, resourcePrivsForApplication.getResourceToResourcePrivileges().values());
+        }
+
+        listener.onResponse(new HasPrivilegesResponse(request.username(), allMatch, cluster,
+            allIndices.getResourceToResourcePrivileges().values(), privilegesByApplication));
+    }
+
+
+    @Override
+    public void getUserPrivileges(Authentication authentication, AuthorizationInfo authorizationInfo, GetUserPrivilegesRequest request,
+                                  ActionListener<GetUserPrivilegesResponse> listener) {
+        if (authorizationInfo instanceof RBACAuthorizationInfo == false) {
+            listener.onFailure(
+                new IllegalArgumentException("unsupported authorization info:" + authorizationInfo.getClass().getSimpleName()));
+        } else {
+            final Role role = ((RBACAuthorizationInfo) authorizationInfo).getRole();
+            listener.onResponse(buildUserPrivilegesResponseObject(role));
+        }
+    }
+
+    GetUserPrivilegesResponse buildUserPrivilegesResponseObject(Role userRole) {
+        logger.trace(() -> new ParameterizedMessage("List privileges for role [{}]", arrayToCommaDelimitedString(userRole.names())));
+
+        // We use sorted sets for Strings because they will typically be small, and having a predictable order allows for simpler testing
+        final Set<String> cluster = new TreeSet<>();
+        // But we don't have a meaningful ordering for objects like ConditionalClusterPrivilege, so the tests work with "random" ordering
+        final Set<ConditionalClusterPrivilege> conditionalCluster = new HashSet<>();
+        for (Tuple<ClusterPrivilege, ConditionalClusterPrivilege> tup : userRole.cluster().privileges()) {
+            if (tup.v2() == null) {
+                if (ClusterPrivilege.NONE.equals(tup.v1()) == false) {
+                    cluster.addAll(tup.v1().name());
+                }
+            } else {
+                conditionalCluster.add(tup.v2());
+            }
+        }
+
+        final Set<GetUserPrivilegesResponse.Indices> indices = new LinkedHashSet<>();
+        for (IndicesPermission.Group group : userRole.indices().groups()) {
+            final Set<BytesReference> queries = group.getQuery() == null ? Collections.emptySet() : group.getQuery();
+            final Set<FieldPermissionsDefinition.FieldGrantExcludeGroup> fieldSecurity = group.getFieldPermissions().hasFieldLevelSecurity()
+                ? group.getFieldPermissions().getFieldPermissionsDefinition().getFieldGrantExcludeGroups() : Collections.emptySet();
+            indices.add(new GetUserPrivilegesResponse.Indices(
+                Arrays.asList(group.indices()),
+                group.privilege().name(),
+                fieldSecurity,
+                queries,
+                group.allowRestrictedIndices()
+            ));
+        }
+
+        final Set<RoleDescriptor.ApplicationResourcePrivileges> application = new LinkedHashSet<>();
+        for (String applicationName : userRole.application().getApplicationNames()) {
+            for (ApplicationPrivilege privilege : userRole.application().getPrivileges(applicationName)) {
+                final Set<String> resources = userRole.application().getResourcePatterns(privilege);
+                if (resources.isEmpty()) {
+                    logger.trace("No resources defined in application privilege {}", privilege);
+                } else {
+                    application.add(RoleDescriptor.ApplicationResourcePrivileges.builder()
+                        .application(applicationName)
+                        .privileges(privilege.name())
+                        .resources(resources)
+                        .build());
+                }
+            }
+        }
+
+        final Privilege runAsPrivilege = userRole.runAs().getPrivilege();
+        final Set<String> runAs;
+        if (Operations.isEmpty(runAsPrivilege.getAutomaton())) {
+            runAs = Collections.emptySet();
+        } else {
+            runAs = runAsPrivilege.name();
+        }
+
+        return new GetUserPrivilegesResponse(cluster, conditionalCluster, indices, application, runAs);
+    }
+
+    static List<String> resolveAuthorizedIndicesFromRole(Role role, String action, Map<String, AliasOrIndex> aliasAndIndexLookup) {
+        Predicate<String> predicate = role.allowedIndicesMatcher(action);
+
+        List<String> indicesAndAliases = new ArrayList<>();
+        // TODO: can this be done smarter? I think there are usually more indices/aliases in the cluster then indices defined a roles?
+        for (Map.Entry<String, AliasOrIndex> entry : aliasAndIndexLookup.entrySet()) {
+            String aliasOrIndex = entry.getKey();
+            if (predicate.test(aliasOrIndex)) {
+                indicesAndAliases.add(aliasOrIndex);
+            }
+        }
+        return Collections.unmodifiableList(indicesAndAliases);
+    }
+
+    private void buildIndicesAccessControl(Authentication authentication, String action,
+                                           AuthorizationInfo authorizationInfo, Set<String> indices,
+                                           Map<String, AliasOrIndex> aliasAndIndexLookup,
+                                           ActionListener<IndexAuthorizationResult> listener) {
+        if (authorizationInfo instanceof RBACAuthorizationInfo) {
+            final Role role = ((RBACAuthorizationInfo) authorizationInfo).getRole();
+            final IndicesAccessControl accessControl = role.authorize(action, indices, aliasAndIndexLookup, fieldPermissionsCache);
+            listener.onResponse(new IndexAuthorizationResult(true, accessControl));
+        } else {
+            listener.onFailure(new IllegalArgumentException("unsupported authorization info:" +
+                authorizationInfo.getClass().getSimpleName()));
+        }
+    }
+
+    private static boolean checkChangePasswordAction(Authentication authentication) {
+        // we need to verify that this user was authenticated by or looked up by a realm type that support password changes
+        // otherwise we open ourselves up to issues where a user in a different realm could be created with the same username
+        // and do malicious things
+        final boolean isRunAs = authentication.getUser().isRunAs();
+        final String realmType;
+        if (isRunAs) {
+            realmType = authentication.getLookedUpBy().getType();
+        } else {
+            realmType = authentication.getAuthenticatedBy().getType();
+        }
+
+        assert realmType != null;
+        // ensure the user was authenticated by a realm that we can change a password for. The native realm is an internal realm and
+        // right now only one can exist in the realm configuration - if this changes we should update this check
+        return ReservedRealm.TYPE.equals(realmType) || NativeRealmSettings.TYPE.equals(realmType);
+    }
+
+    static class RBACAuthorizationInfo implements AuthorizationInfo {
+
+        private final Role role;
+        private final Map<String, Object> info;
+        private final RBACAuthorizationInfo authenticatedUserAuthorizationInfo;
+
+        RBACAuthorizationInfo(Role role, Role authenticatedUserRole) {
+            this.role = role;
+            this.info = Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, role.names());
+            this.authenticatedUserAuthorizationInfo =
+                authenticatedUserRole == null ? this : new RBACAuthorizationInfo(authenticatedUserRole, null);
+        }
+
+        Role getRole() {
+            return role;
+        }
+
+        @Override
+        public Map<String, Object> asMap() {
+            return info;
+        }
+
+        @Override
+        public RBACAuthorizationInfo getAuthenticatedUserAuthorizationInfo() {
+            return authenticatedUserAuthorizationInfo;
+        }
+    }
+
+    private static boolean isScrollRelatedAction(String action) {
+        return action.equals(SearchScrollAction.NAME) ||
+            action.equals(SearchTransportService.FETCH_ID_SCROLL_ACTION_NAME) ||
+            action.equals(SearchTransportService.QUERY_FETCH_SCROLL_ACTION_NAME) ||
+            action.equals(SearchTransportService.QUERY_SCROLL_ACTION_NAME) ||
+            action.equals(SearchTransportService.FREE_CONTEXT_SCROLL_ACTION_NAME) ||
+            action.equals(ClearScrollAction.NAME) ||
+            action.equals("indices:data/read/sql/close_cursor") ||
+            action.equals(SearchTransportService.CLEAR_SCROLL_CONTEXTS_ACTION_NAME);
+    }
+}

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListener.java

@@ -16,9 +16,10 @@ import org.elasticsearch.xpack.security.audit.AuditTrailService;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationField;
 import org.elasticsearch.xpack.security.audit.AuditUtil;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationInfo;
 
+import static org.elasticsearch.xpack.security.authz.AuthorizationService.AUTHORIZATION_INFO_KEY;
 import static org.elasticsearch.xpack.security.authz.AuthorizationService.ORIGINATING_ACTION_KEY;
-import static org.elasticsearch.xpack.security.authz.AuthorizationService.ROLE_NAMES_KEY;
 
 /**
  * A {@link SearchOperationListener} that is used to provide authorization for scroll requests.
@@ -64,7 +65,7 @@ public final class SecuritySearchOperationListener implements SearchOperationLis
                 final Authentication current = Authentication.getAuthentication(threadContext);
                 final String action = threadContext.getTransient(ORIGINATING_ACTION_KEY);
                 ensureAuthenticatedUserIsSame(originalAuth, current, auditTrailService, searchContext.id(), action, request,
-                        AuditUtil.extractRequestId(threadContext), threadContext.getTransient(ROLE_NAMES_KEY));
+                        AuditUtil.extractRequestId(threadContext), threadContext.getTransient(AUTHORIZATION_INFO_KEY));
             }
         }
     }
@@ -76,7 +77,8 @@ public final class SecuritySearchOperationListener implements SearchOperationLis
      * (or lookup) realm. To work around this we compare the username and the originating realm type.
      */
     static void ensureAuthenticatedUserIsSame(Authentication original, Authentication current, AuditTrailService auditTrailService,
-                                              long id, String action, TransportRequest request, String requestId, String[] roleNames) {
+                                              long id, String action, TransportRequest request, String requestId,
+                                              AuthorizationInfo authorizationInfo) {
         // this is really a best effort attempt since we cannot guarantee principal uniqueness
         // and realm names can change between nodes.
         final boolean samePrincipal = original.getUser().principal().equals(current.getUser().principal());
@@ -95,7 +97,7 @@ public final class SecuritySearchOperationListener implements SearchOperationLis
 
         final boolean sameUser = samePrincipal && sameRealmType;
         if (sameUser == false) {
-            auditTrailService.accessDenied(requestId, current, action, request, roleNames);
+            auditTrailService.accessDenied(requestId, current, action, request, authorizationInfo);
             throw new SearchContextMissingException(id);
         }
     }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/interceptor/BulkShardRequestInterceptor.java → x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/BulkShardRequestInterceptor.java

@@ -3,11 +3,12 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.security.action.interceptor;
+package org.elasticsearch.xpack.security.authz.interceptor;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.bulk.BulkItemRequest;
 import org.elasticsearch.action.bulk.BulkShardRequest;
 import org.elasticsearch.action.update.UpdateRequest;
@@ -15,16 +16,16 @@ import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.TransportRequest;
-import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationInfo;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.RequestInfo;
 import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField;
 import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
-import org.elasticsearch.xpack.core.security.authz.permission.Role;
 
 /**
  * Similar to {@link UpdateRequestInterceptor}, but checks if there are update requests embedded in a bulk request.
  */
-public class BulkShardRequestInterceptor implements RequestInterceptor<BulkShardRequest> {
+public class BulkShardRequestInterceptor implements RequestInterceptor {
 
     private static final Logger logger = LogManager.getLogger(BulkShardRequestInterceptor.class);
 
@@ -37,31 +38,36 @@ public class BulkShardRequestInterceptor implements RequestInterceptor<BulkShard
     }
 
     @Override
-    public void intercept(BulkShardRequest request, Authentication authentication, Role userPermissions, String action) {
-        if (licenseState.isDocumentAndFieldLevelSecurityAllowed()) {
+    public void intercept(RequestInfo requestInfo, AuthorizationEngine authzEngine, AuthorizationInfo authorizationInfo,
+                          ActionListener<Void> listener) {
+        if (requestInfo.getRequest() instanceof BulkShardRequest && licenseState.isDocumentAndFieldLevelSecurityAllowed()) {
             IndicesAccessControl indicesAccessControl = threadContext.getTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY);
 
-            for (BulkItemRequest bulkItemRequest : request.items()) {
+            final BulkShardRequest bulkShardRequest = (BulkShardRequest) requestInfo.getRequest();
+            for (BulkItemRequest bulkItemRequest : bulkShardRequest.items()) {
                 IndicesAccessControl.IndexAccessControl indexAccessControl =
                     indicesAccessControl.getIndexPermissions(bulkItemRequest.index());
+                boolean found = false;
                 if (indexAccessControl != null) {
                     boolean fls = indexAccessControl.getFieldPermissions().hasFieldLevelSecurity();
                     boolean dls = indexAccessControl.getDocumentPermissions().hasDocumentLevelPermissions();
                     if (fls || dls) {
                         if (bulkItemRequest.request() instanceof UpdateRequest) {
-                            throw new ElasticsearchSecurityException("Can't execute a bulk request with update requests embedded if " +
-                                "field or document level security is enabled", RestStatus.BAD_REQUEST);
+                            found = true;
+                            logger.trace("aborting bulk item update request for index [{}]", bulkItemRequest.index());
+                            bulkItemRequest.abort(bulkItemRequest.index(), new ElasticsearchSecurityException("Can't execute a bulk " +
+                                "item request with update requests embedded if field or document level security is enabled",
+                                RestStatus.BAD_REQUEST));
                         }
                     }
                 }
-                logger.trace("intercepted bulk request for index [{}] without any update requests, continuing execution",
-                    bulkItemRequest.index());
+
+                if (found == false) {
+                    logger.trace("intercepted bulk request for index [{}] without any update requests, continuing execution",
+                        bulkItemRequest.index());
+                }
             }
         }
-    }
-
-    @Override
-    public boolean supports(TransportRequest request) {
-        return request instanceof BulkShardRequest;
+        listener.onResponse(null);
     }
 }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/FieldAndDocumentLevelSecurityRequestInterceptor.java

@@ -0,0 +1,68 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.authz.interceptor;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.IndicesRequest;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationInfo;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.RequestInfo;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField;
+import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
+
+/**
+ * Base class for interceptors that disables features when field level security is configured for indices a request
+ * is going to execute on.
+ */
+abstract class FieldAndDocumentLevelSecurityRequestInterceptor implements RequestInterceptor {
+
+    private final ThreadContext threadContext;
+    private final XPackLicenseState licenseState;
+    private final Logger logger;
+
+    FieldAndDocumentLevelSecurityRequestInterceptor(ThreadContext threadContext, XPackLicenseState licenseState) {
+        this.threadContext = threadContext;
+        this.licenseState = licenseState;
+        this.logger = LogManager.getLogger(getClass());
+    }
+
+    @Override
+    public void intercept(RequestInfo requestInfo, AuthorizationEngine authorizationEngine, AuthorizationInfo authorizationInfo,
+                          ActionListener<Void> listener) {
+        if (requestInfo.getRequest() instanceof IndicesRequest) {
+            IndicesRequest indicesRequest = (IndicesRequest) requestInfo.getRequest();
+            if (supports(indicesRequest) && licenseState.isDocumentAndFieldLevelSecurityAllowed()) {
+                final IndicesAccessControl indicesAccessControl =
+                    threadContext.getTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY);
+                for (String index : indicesRequest.indices()) {
+                    IndicesAccessControl.IndexAccessControl indexAccessControl = indicesAccessControl.getIndexPermissions(index);
+                    if (indexAccessControl != null) {
+                        boolean fieldLevelSecurityEnabled = indexAccessControl.getFieldPermissions().hasFieldLevelSecurity();
+                        boolean documentLevelSecurityEnabled = indexAccessControl.getDocumentPermissions().hasDocumentLevelPermissions();
+                        if (fieldLevelSecurityEnabled || documentLevelSecurityEnabled) {
+                            logger.trace("intercepted request for index [{}] with field level access controls [{}] " +
+                                "document level access controls [{}]. disabling conflicting features",
+                                index, fieldLevelSecurityEnabled, documentLevelSecurityEnabled);
+                            disableFeatures(indicesRequest, fieldLevelSecurityEnabled, documentLevelSecurityEnabled, listener);
+                            return;
+                        }
+                    }
+                    logger.trace("intercepted request for index [{}] without field or document level access controls", index);
+                }
+            }
+        }
+        listener.onResponse(null);
+    }
+
+    abstract void disableFeatures(IndicesRequest request, boolean fieldLevelSecurityEnabled, boolean documentLevelSecurityEnabled,
+                                  ActionListener<Void> listener);
+
+    abstract boolean supports(IndicesRequest request);
+}

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/IndicesAliasesRequestInterceptor.java

@@ -0,0 +1,105 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.authz.interceptor;
+
+import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationInfo;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.RequestInfo;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField;
+import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
+import org.elasticsearch.xpack.core.security.support.Exceptions;
+import org.elasticsearch.xpack.security.audit.AuditTrailService;
+import org.elasticsearch.xpack.security.audit.AuditUtil;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.action.support.ContextPreservingActionListener.wrapPreservingContext;
+
+public final class IndicesAliasesRequestInterceptor implements RequestInterceptor {
+
+    private final ThreadContext threadContext;
+    private final XPackLicenseState licenseState;
+    private final AuditTrailService auditTrailService;
+
+    public IndicesAliasesRequestInterceptor(ThreadContext threadContext, XPackLicenseState licenseState,
+                                            AuditTrailService auditTrailService) {
+        this.threadContext = threadContext;
+        this.licenseState = licenseState;
+        this.auditTrailService = auditTrailService;
+    }
+
+    @Override
+    public void intercept(RequestInfo requestInfo, AuthorizationEngine authorizationEngine, AuthorizationInfo authorizationInfo,
+                          ActionListener<Void> listener) {
+        if (requestInfo.getRequest() instanceof IndicesAliasesRequest) {
+            final IndicesAliasesRequest request = (IndicesAliasesRequest) requestInfo.getRequest();
+            final XPackLicenseState frozenLicenseState = licenseState.copyCurrentLicenseState();
+            if (frozenLicenseState.isAuthAllowed()) {
+                if (frozenLicenseState.isDocumentAndFieldLevelSecurityAllowed()) {
+                    IndicesAccessControl indicesAccessControl =
+                        threadContext.getTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY);
+                    for (IndicesAliasesRequest.AliasActions aliasAction : request.getAliasActions()) {
+                        if (aliasAction.actionType() == IndicesAliasesRequest.AliasActions.Type.ADD) {
+                            for (String index : aliasAction.indices()) {
+                                IndicesAccessControl.IndexAccessControl indexAccessControl =
+                                    indicesAccessControl.getIndexPermissions(index);
+                                if (indexAccessControl != null) {
+                                    final boolean fls = indexAccessControl.getFieldPermissions().hasFieldLevelSecurity();
+                                    final boolean dls = indexAccessControl.getDocumentPermissions().hasDocumentLevelPermissions();
+                                    if (fls || dls) {
+                                        listener.onFailure(new ElasticsearchSecurityException("Alias requests are not allowed for " +
+                                            "users who have field or document level security enabled on one of the indices",
+                                            RestStatus.BAD_REQUEST));
+                                        return;
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+
+            Map<String, List<String>> indexToAliasesMap = request.getAliasActions().stream()
+                .filter(aliasAction -> aliasAction.actionType() == IndicesAliasesRequest.AliasActions.Type.ADD)
+                .flatMap(aliasActions ->
+                    Arrays.stream(aliasActions.indices())
+                        .map(indexName -> new Tuple<>(indexName, Arrays.asList(aliasActions.aliases()))))
+                .collect(Collectors.toMap(Tuple::v1, Tuple::v2, (existing, toMerge) -> {
+                    List<String> list = new ArrayList<>(existing.size() + toMerge.size());
+                    list.addAll(existing);
+                    list.addAll(toMerge);
+                    return list;
+                }));
+            authorizationEngine.validateIndexPermissionsAreSubset(requestInfo, authorizationInfo, indexToAliasesMap,
+                wrapPreservingContext(ActionListener.wrap(authzResult -> {
+                    if (authzResult.isGranted()) {
+                        // do not audit success again
+                        listener.onResponse(null);
+                    } else {
+                        auditTrailService.accessDenied(AuditUtil.extractRequestId(threadContext), requestInfo.getAuthentication(),
+                            requestInfo.getAction(), request, authorizationInfo);
+                        listener.onFailure(Exceptions.authorizationError("Adding an alias is not allowed when the alias " +
+                            "has more permissions than any of the indices"));
+                    }
+                }, listener::onFailure), threadContext));
+            } else {
+                listener.onResponse(null);
+            }
+        } else {
+            listener.onResponse(null);
+        }
+    }
+}

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/RequestInterceptor.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.authz.interceptor;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationInfo;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.RequestInfo;
+
+/**
+ * A request interceptor can introspect a request and modify it.
+ */
+public interface RequestInterceptor {
+
+    /**
+     * This interceptor will introspect the request and potentially modify it. If the interceptor does not apply
+     * to the request then the request will not be modified.
+     */
+    void intercept(RequestInfo requestInfo, AuthorizationEngine authorizationEngine, AuthorizationInfo authorizationInfo,
+                   ActionListener<Void> listener);
+}

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/ResizeRequestInterceptor.java

@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.authz.interceptor;
+
+import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.shrink.ResizeRequest;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationInfo;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.RequestInfo;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField;
+import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
+import org.elasticsearch.xpack.core.security.support.Exceptions;
+import org.elasticsearch.xpack.security.audit.AuditTrailService;
+
+import java.util.Collections;
+
+import static org.elasticsearch.action.support.ContextPreservingActionListener.wrapPreservingContext;
+import static org.elasticsearch.xpack.security.audit.AuditUtil.extractRequestId;
+
+public final class ResizeRequestInterceptor implements RequestInterceptor {
+
+    private final ThreadContext threadContext;
+    private final XPackLicenseState licenseState;
+    private final AuditTrailService auditTrailService;
+
+    public ResizeRequestInterceptor(ThreadPool threadPool, XPackLicenseState licenseState,
+                                    AuditTrailService auditTrailService) {
+        this.threadContext = threadPool.getThreadContext();
+        this.licenseState = licenseState;
+        this.auditTrailService = auditTrailService;
+    }
+
+    @Override
+    public void intercept(RequestInfo requestInfo, AuthorizationEngine authorizationEngine, AuthorizationInfo authorizationInfo,
+                          ActionListener<Void> listener) {
+        if (requestInfo.getRequest() instanceof ResizeRequest) {
+            final ResizeRequest request = (ResizeRequest) requestInfo.getRequest();
+            final XPackLicenseState frozenLicenseState = licenseState.copyCurrentLicenseState();
+            if (frozenLicenseState.isAuthAllowed()) {
+                if (frozenLicenseState.isDocumentAndFieldLevelSecurityAllowed()) {
+                    IndicesAccessControl indicesAccessControl =
+                        threadContext.getTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY);
+                    IndicesAccessControl.IndexAccessControl indexAccessControl =
+                        indicesAccessControl.getIndexPermissions(request.getSourceIndex());
+                    if (indexAccessControl != null) {
+                        final boolean fls = indexAccessControl.getFieldPermissions().hasFieldLevelSecurity();
+                        final boolean dls = indexAccessControl.getDocumentPermissions().hasDocumentLevelPermissions();
+                        if (fls || dls) {
+                            listener.onFailure(new ElasticsearchSecurityException("Resize requests are not allowed for users when " +
+                                "field or document level security is enabled on the source index", RestStatus.BAD_REQUEST));
+                            return;
+                        }
+                    }
+                }
+
+                authorizationEngine.validateIndexPermissionsAreSubset(requestInfo, authorizationInfo,
+                    Collections.singletonMap(request.getSourceIndex(), Collections.singletonList(request.getTargetIndexRequest().index())),
+                    wrapPreservingContext(ActionListener.wrap(authzResult -> {
+                        if (authzResult.isGranted()) {
+                            listener.onResponse(null);
+                        } else {
+                            if (authzResult.isAuditable()) {
+                                auditTrailService.accessDenied(extractRequestId(threadContext), requestInfo.getAuthentication(),
+                                    requestInfo.getAction(), request, authorizationInfo);
+                            }
+                            listener.onFailure(Exceptions.authorizationError("Resizing an index is not allowed when the target index " +
+                                "has more permissions than the source index"));
+                        }
+                    }, listener::onFailure), threadContext));
+            } else {
+                listener.onResponse(null);
+            }
+        } else {
+            listener.onResponse(null);
+        }
+    }
+}

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/interceptor/SearchRequestInterceptor.java → x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/SearchRequestInterceptor.java

@@ -3,42 +3,48 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.security.action.interceptor;
+package org.elasticsearch.xpack.security.authz.interceptor;
 
 import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.TransportRequest;
 
 /**
  * If field level security is enabled this interceptor disables the request cache for search requests.
  */
-public class SearchRequestInterceptor extends FieldAndDocumentLevelSecurityRequestInterceptor<SearchRequest> {
+public class SearchRequestInterceptor extends FieldAndDocumentLevelSecurityRequestInterceptor {
 
     public SearchRequestInterceptor(ThreadPool threadPool, XPackLicenseState licenseState) {
         super(threadPool.getThreadContext(), licenseState);
     }
 
     @Override
-    public void disableFeatures(SearchRequest request, boolean fieldLevelSecurityEnabled, boolean documentLevelSecurityEnabled) {
+    public void disableFeatures(IndicesRequest indicesRequest, boolean fieldLevelSecurityEnabled, boolean documentLevelSecurityEnabled,
+                                ActionListener<Void> listener) {
+        final SearchRequest request = (SearchRequest) indicesRequest;
         request.requestCache(false);
 
         if (documentLevelSecurityEnabled) {
             if (request.source() != null && request.source().suggest() != null) {
-                throw new ElasticsearchSecurityException("Suggest isn't supported if document level security is enabled",
-                        RestStatus.BAD_REQUEST);
-            }
-            if (request.source() != null && request.source().profile()) {
-                throw new ElasticsearchSecurityException("A search request cannot be profiled if document level security is enabled",
-                        RestStatus.BAD_REQUEST);
+                listener.onFailure(new ElasticsearchSecurityException("Suggest isn't supported if document level security is enabled",
+                        RestStatus.BAD_REQUEST));
+            } else if (request.source() != null && request.source().profile()) {
+                listener.onFailure(new ElasticsearchSecurityException("A search request cannot be profiled if document level security " +
+                    "is enabled", RestStatus.BAD_REQUEST));
+            } else {
+                listener.onResponse(null);
             }
+        } else {
+            listener.onResponse(null);
         }
     }
 
     @Override
-    public boolean supports(TransportRequest request) {
+    public boolean supports(IndicesRequest request) {
         return request instanceof SearchRequest;
     }
 }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/interceptor/UpdateRequestInterceptor.java → x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/UpdateRequestInterceptor.java

@@ -3,14 +3,15 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.security.action.interceptor;
+package org.elasticsearch.xpack.security.authz.interceptor;
 
 import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.TransportRequest;
 
 /**
  * A request interceptor that fails update request if field or document level security is enabled.
@@ -19,20 +20,21 @@ import org.elasticsearch.transport.TransportRequest;
  * because only the fields that a role can see would be used to perform the update and without knowing the user may
  * remove the other fields, not visible for him, from the document being updated.
  */
-public class UpdateRequestInterceptor extends FieldAndDocumentLevelSecurityRequestInterceptor<UpdateRequest> {
+public class UpdateRequestInterceptor extends FieldAndDocumentLevelSecurityRequestInterceptor {
 
     public UpdateRequestInterceptor(ThreadPool threadPool, XPackLicenseState licenseState) {
         super(threadPool.getThreadContext(), licenseState);
     }
 
     @Override
-    protected void disableFeatures(UpdateRequest updateRequest, boolean fieldLevelSecurityEnabled, boolean documentLevelSecurityEnabled) {
-        throw new ElasticsearchSecurityException("Can't execute an update request if field or document level security is enabled",
-                RestStatus.BAD_REQUEST);
+    protected void disableFeatures(IndicesRequest updateRequest, boolean fieldLevelSecurityEnabled, boolean documentLevelSecurityEnabled,
+                                   ActionListener<Void> listener) {
+        listener.onFailure(new ElasticsearchSecurityException("Can't execute an update request if field or document level security " +
+            "is enabled", RestStatus.BAD_REQUEST));
     }
 
     @Override
-    public boolean supports(TransportRequest request) {
+    public boolean supports(IndicesRequest request) {
         return request instanceof UpdateRequest;
     }
 }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java

@@ -25,11 +25,13 @@ import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.xpack.core.common.IteratingActionListener;
+import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.IndicesPrivileges;
 import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsCache;
 import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsDefinition;
 import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsDefinition.FieldGrantExcludeGroup;
+import org.elasticsearch.xpack.core.security.authz.permission.LimitedRole;
 import org.elasticsearch.xpack.core.security.authz.permission.Role;
 import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege;
 import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivilege;
@@ -37,6 +39,12 @@ import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege;
 import org.elasticsearch.xpack.core.security.authz.privilege.Privilege;
 import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
 import org.elasticsearch.xpack.core.security.authz.store.RoleRetrievalResult;
+import org.elasticsearch.xpack.core.security.user.AnonymousUser;
+import org.elasticsearch.xpack.core.security.user.SystemUser;
+import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.xpack.core.security.user.XPackSecurityUser;
+import org.elasticsearch.xpack.core.security.user.XPackUser;
+import org.elasticsearch.xpack.security.authc.ApiKeyService;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 
 import java.util.ArrayList;
@@ -97,19 +105,24 @@ public class CompositeRolesStore {
     private final Cache<String, Boolean> negativeLookupCache;
     private final ThreadContext threadContext;
     private final AtomicLong numInvalidation = new AtomicLong();
+    private final AnonymousUser anonymousUser;
+    private final ApiKeyService apiKeyService;
+    private final boolean isAnonymousEnabled;
     private final List<BiConsumer<Set<String>, ActionListener<RoleRetrievalResult>>> builtInRoleProviders;
     private final List<BiConsumer<Set<String>, ActionListener<RoleRetrievalResult>>> allRoleProviders;
 
     public CompositeRolesStore(Settings settings, FileRolesStore fileRolesStore, NativeRolesStore nativeRolesStore,
                                ReservedRolesStore reservedRolesStore, NativePrivilegeStore privilegeStore,
                                List<BiConsumer<Set<String>, ActionListener<RoleRetrievalResult>>> rolesProviders,
-                               ThreadContext threadContext, XPackLicenseState licenseState, FieldPermissionsCache fieldPermissionsCache) {
+                               ThreadContext threadContext, XPackLicenseState licenseState, FieldPermissionsCache fieldPermissionsCache,
+                               ApiKeyService apiKeyService) {
         this.fileRolesStore = fileRolesStore;
         fileRolesStore.addListener(this::invalidate);
         this.nativeRolesStore = nativeRolesStore;
         this.privilegeStore = privilegeStore;
         this.licenseState = licenseState;
         this.fieldPermissionsCache = fieldPermissionsCache;
+        this.apiKeyService = apiKeyService;
         CacheBuilder<RoleKey, Role> builder = CacheBuilder.builder();
         final int cacheSize = CACHE_SIZE_SETTING.get(settings);
         if (cacheSize >= 0) {
@@ -133,6 +146,8 @@ public class CompositeRolesStore {
             allList.addAll(rolesProviders);
             this.allRoleProviders = Collections.unmodifiableList(allList);
         }
+        this.anonymousUser = new AnonymousUser(settings);
+        this.isAnonymousEnabled = AnonymousUser.isAnonymousEnabled(settings);
     }
 
     public void roles(Set<String> roleNames, ActionListener<Role> roleActionListener) {
@@ -164,6 +179,60 @@ public class CompositeRolesStore {
         }
     }
 
+    public void getRoles(User user, Authentication authentication, ActionListener<Role> roleActionListener) {
+        // we need to special case the internal users in this method, if we apply the anonymous roles to every user including these system
+        // user accounts then we run into the chance of a deadlock because then we need to get a role that we may be trying to get as the
+        // internal user. The SystemUser is special cased as it has special privileges to execute internal actions and should never be
+        // passed into this method. The XPackUser has the Superuser role and we can simply return that
+        if (SystemUser.is(user)) {
+            throw new IllegalArgumentException("the user [" + user.principal() + "] is the system user and we should never try to get its" +
+                " roles");
+        }
+        if (XPackUser.is(user)) {
+            assert XPackUser.INSTANCE.roles().length == 1;
+            roleActionListener.onResponse(XPackUser.ROLE);
+            return;
+        }
+        if (XPackSecurityUser.is(user)) {
+            roleActionListener.onResponse(ReservedRolesStore.SUPERUSER_ROLE);
+            return;
+        }
+
+        final Authentication.AuthenticationType authType = authentication.getAuthenticationType();
+        if (authType == Authentication.AuthenticationType.API_KEY) {
+            apiKeyService.getRoleForApiKey(authentication, ActionListener.wrap(apiKeyRoleDescriptors -> {
+                final List<RoleDescriptor> descriptors = apiKeyRoleDescriptors.getRoleDescriptors();
+                if (descriptors == null) {
+                    roleActionListener.onFailure(new IllegalStateException("missing role descriptors"));
+                } else if (apiKeyRoleDescriptors.getLimitedByRoleDescriptors() == null) {
+                    buildAndCacheRoleFromDescriptors(descriptors, apiKeyRoleDescriptors.getApiKeyId() + "_role_desc", roleActionListener);
+                } else {
+                    buildAndCacheRoleFromDescriptors(descriptors, apiKeyRoleDescriptors.getApiKeyId() + "_role_desc",
+                        ActionListener.wrap(role -> buildAndCacheRoleFromDescriptors(apiKeyRoleDescriptors.getLimitedByRoleDescriptors(),
+                            apiKeyRoleDescriptors.getApiKeyId() + "_limited_role_desc", ActionListener.wrap(
+                                limitedBy -> roleActionListener.onResponse(LimitedRole.createLimitedRole(role, limitedBy)),
+                                roleActionListener::onFailure)), roleActionListener::onFailure));
+                }
+            }, roleActionListener::onFailure));
+        } else {
+            Set<String> roleNames = new HashSet<>(Arrays.asList(user.roles()));
+            if (isAnonymousEnabled && anonymousUser.equals(user) == false) {
+                if (anonymousUser.roles().length == 0) {
+                    throw new IllegalStateException("anonymous is only enabled when the anonymous user has roles");
+                }
+                Collections.addAll(roleNames, anonymousUser.roles());
+            }
+
+            if (roleNames.isEmpty()) {
+                roleActionListener.onResponse(Role.EMPTY);
+            } else if (roleNames.contains(ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR.getName())) {
+                roleActionListener.onResponse(ReservedRolesStore.SUPERUSER_ROLE);
+            } else {
+                roles(roleNames, roleActionListener);
+            }
+        }
+    }
+
     public void buildAndCacheRoleFromDescriptors(Collection<RoleDescriptor> roleDescriptors, String source,
                                                   ActionListener<Role> listener) {
         if (ROLES_STORE_SOURCE.equals(source)) {

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/ServerTransportFilter.java

@@ -30,7 +30,6 @@ import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.security.action.SecurityActionMapper;
 import org.elasticsearch.xpack.security.authc.AuthenticationService;
 import org.elasticsearch.xpack.security.authz.AuthorizationService;
-import org.elasticsearch.xpack.security.authz.AuthorizationUtils;
 
 import java.io.IOException;
 
@@ -121,20 +120,10 @@ public interface ServerTransportFilter {
                     SystemUser.is(authentication.getUser()) == false) {
                     securityContext.executeAsUser(SystemUser.INSTANCE, (ctx) -> {
                         final Authentication replaced = Authentication.getAuthentication(threadContext);
-                        final AuthorizationUtils.AsyncAuthorizer asyncAuthorizer =
-                            new AuthorizationUtils.AsyncAuthorizer(replaced, listener, (userRoles, runAsRoles) -> {
-                                authzService.authorize(replaced, securityAction, request, userRoles, runAsRoles);
-                                listener.onResponse(null);
-                            });
-                        asyncAuthorizer.authorize(authzService);
+                        authzService.authorize(replaced, securityAction, request, listener);
                     }, version);
                 } else {
-                    final AuthorizationUtils.AsyncAuthorizer asyncAuthorizer =
-                        new AuthorizationUtils.AsyncAuthorizer(authentication, listener, (userRoles, runAsRoles) -> {
-                            authzService.authorize(authentication, securityAction, request, userRoles, runAsRoles);
-                            listener.onResponse(null);
-                        });
-                    asyncAuthorizer.authorize(authzService);
+                    authzService.authorize(authentication, securityAction, request, listener);
                 }
             }, listener::onFailure));
         }

x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java

@@ -841,7 +841,7 @@ public class DocumentLevelSecurityTests extends SecurityIntegTestCase {
         ElasticsearchSecurityException securityException = (ElasticsearchSecurityException) bulkItem.getFailure().getCause();
         assertThat(securityException.status(), equalTo(RestStatus.BAD_REQUEST));
         assertThat(securityException.getMessage(),
-                equalTo("Can't execute a bulk request with update requests embedded if field or document level security is enabled"));
+                equalTo("Can't execute a bulk item request with update requests embedded if field or document level security is enabled"));
 
         assertThat(client().prepareGet("test", "type", "1").get().getSource().get("field1").toString(), equalTo("value2"));
 

x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java

@@ -1448,7 +1448,7 @@ public class FieldLevelSecurityTests extends SecurityIntegTestCase {
         ElasticsearchSecurityException securityException = (ElasticsearchSecurityException) bulkItem.getFailure().getCause();
         assertThat(securityException.status(), equalTo(RestStatus.BAD_REQUEST));
         assertThat(securityException.getMessage(),
-                equalTo("Can't execute a bulk request with update requests embedded if field or document level security is enabled"));
+                equalTo("Can't execute a bulk item request with update requests embedded if field or document level security is enabled"));
 
         assertThat(client().prepareGet("test", "type", "1").get().getSource().get("field2").toString(), equalTo("value2"));
 

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/filter/SecurityActionFilterTests.java

@@ -27,11 +27,11 @@ import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.xpack.core.security.SecurityContext;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationField;
-import org.elasticsearch.xpack.core.security.authz.permission.Role;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
 import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.security.authc.AuthenticationService;
@@ -39,7 +39,6 @@ import org.elasticsearch.xpack.security.authz.AuthorizationService;
 import org.junit.Before;
 
 import java.util.Collections;
-import java.util.HashSet;
 
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.eq;
@@ -83,8 +82,7 @@ public class SecurityActionFilterTests extends ESTestCase {
         when(state.nodes()).thenReturn(nodes);
 
         SecurityContext securityContext = new SecurityContext(settings, threadContext);
-        filter = new SecurityActionFilter(authcService, authzService,
-                        licenseState, new HashSet<>(), threadPool, securityContext, destructiveOperations);
+        filter = new SecurityActionFilter(authcService, authzService, licenseState, threadPool, securityContext, destructiveOperations);
     }
 
     public void testApply() throws Exception {
@@ -100,15 +98,14 @@ public class SecurityActionFilterTests extends ESTestCase {
             callback.onResponse(authentication);
             return Void.TYPE;
         }).when(authcService).authenticate(eq("_action"), eq(request), eq(SystemUser.INSTANCE), any(ActionListener.class));
-        final Role empty = Role.EMPTY;
         doAnswer((i) -> {
-            ActionListener callback =
-                    (ActionListener) i.getArguments()[2];
-            callback.onResponse(empty);
+            ActionListener<Void> callback = (ActionListener<Void>) i.getArguments()[3];
+            callback.onResponse(null);
             return Void.TYPE;
-        }).when(authzService).roles(any(User.class), any(Authentication.class), any(ActionListener.class));
+        }).when(authzService)
+            .authorize(any(Authentication.class), any(String.class), any(TransportRequest.class), any(ActionListener.class));
         filter.apply(task, "_action", request, listener, chain);
-        verify(authzService).authorize(authentication, "_action", request, empty, null);
+        verify(authzService).authorize(eq(authentication), eq("_action"), eq(request), any(ActionListener.class));
         verify(chain).proceed(eq(task), eq("_action"), eq(request), isA(ContextPreservingActionListener.class));
     }
 
@@ -127,20 +124,18 @@ public class SecurityActionFilterTests extends ESTestCase {
             callback.onResponse(authentication);
             return Void.TYPE;
         }).when(authcService).authenticate(eq("_action"), eq(request), eq(SystemUser.INSTANCE), any(ActionListener.class));
-        final Role empty = Role.EMPTY;
         doAnswer((i) -> {
-            ActionListener callback =
-                    (ActionListener) i.getArguments()[2];
-            assertEquals(authentication, threadContext.getTransient(AuthenticationField.AUTHENTICATION_KEY));
-            callback.onResponse(empty);
+            ActionListener<Void> callback = (ActionListener<Void>) i.getArguments()[3];
+            callback.onResponse(null);
             return Void.TYPE;
-        }).when(authzService).roles(any(User.class), any(Authentication.class), any(ActionListener.class));
+        }).when(authzService)
+            .authorize(any(Authentication.class), any(String.class), any(TransportRequest.class), any(ActionListener.class));
         assertNull(threadContext.getTransient(AuthenticationField.AUTHENTICATION_KEY));
 
         filter.apply(task, "_action", request, listener, chain);
 
         assertNull(threadContext.getTransient(AuthenticationField.AUTHENTICATION_KEY));
-        verify(authzService).authorize(authentication, "_action", request, empty, null);
+        verify(authzService).authorize(eq(authentication), eq("_action"), eq(request), any(ActionListener.class));
         verify(chain).proceed(eq(task), eq("_action"), eq(request), isA(ContextPreservingActionListener.class));
     }
 
@@ -169,6 +164,12 @@ public class SecurityActionFilterTests extends ESTestCase {
             callback.onResponse(threadContext.getTransient(AuthenticationField.AUTHENTICATION_KEY));
             return Void.TYPE;
         }).when(authcService).authenticate(eq(action), eq(request), eq(SystemUser.INSTANCE), any(ActionListener.class));
+        doAnswer((i) -> {
+            ActionListener<Void> callback = (ActionListener<Void>) i.getArguments()[3];
+            callback.onResponse(null);
+            return Void.TYPE;
+        }).when(authzService)
+            .authorize(any(Authentication.class), any(String.class), any(TransportRequest.class), any(ActionListener.class));
 
         filter.apply(task, action, request, listener, chain);
 
@@ -198,19 +199,18 @@ public class SecurityActionFilterTests extends ESTestCase {
             callback.onResponse(authentication);
             return Void.TYPE;
         }).when(authcService).authenticate(eq(action), eq(request), eq(SystemUser.INSTANCE), any(ActionListener.class));
-        final Role empty = Role.EMPTY;
         doAnswer((i) -> {
-            ActionListener callback =
-                    (ActionListener) i.getArguments()[2];
-            callback.onResponse(empty);
+            ActionListener<Void> callback = (ActionListener<Void>) i.getArguments()[3];
+            callback.onResponse(null);
             return Void.TYPE;
-        }).when(authzService).roles(any(User.class), any(Authentication.class), any(ActionListener.class));
+        }).when(authzService)
+            .authorize(any(Authentication.class), any(String.class), any(TransportRequest.class), any(ActionListener.class));
         filter.apply(task, action, request, listener, chain);
         if (failDestructiveOperations) {
             verify(listener).onFailure(isA(IllegalArgumentException.class));
             verifyNoMoreInteractions(authzService, chain);
         } else {
-            verify(authzService).authorize(authentication, action, request, empty, null);
+            verify(authzService).authorize(eq(authentication), eq(action), eq(request), any(ActionListener.class));
             verify(chain).proceed(eq(task), eq(action), eq(request), isA(ContextPreservingActionListener.class));
         }
     }
@@ -229,14 +229,7 @@ public class SecurityActionFilterTests extends ESTestCase {
             callback.onResponse(authentication);
             return Void.TYPE;
         }).when(authcService).authenticate(eq("_action"), eq(request), eq(SystemUser.INSTANCE), any(ActionListener.class));
-        doAnswer((i) -> {
-            ActionListener callback =
-                    (ActionListener) i.getArguments()[2];
-            callback.onResponse(Role.EMPTY);
-            return Void.TYPE;
-        }).when(authzService).roles(any(User.class), any(Authentication.class), any(ActionListener.class));
-        doThrow(exception).when(authzService).authorize(eq(authentication), eq("_action"), eq(request), any(Role.class),
-                any(Role.class));
+        doThrow(exception).when(authzService).authorize(eq(authentication), eq("_action"), eq(request), any(ActionListener.class));
         filter.apply(task, "_action", request, listener, chain);
         verify(listener).onFailure(exception);
         verifyNoMoreInteractions(chain);

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportGetUserPrivilegesActionTests.java

@@ -1,86 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-package org.elasticsearch.xpack.security.action.user;
-
-import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.util.set.Sets;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.xpack.core.security.action.user.GetUserPrivilegesResponse;
-import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
-import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissions;
-import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsDefinition;
-import org.elasticsearch.xpack.core.security.authz.permission.Role;
-import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege;
-import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivileges.ManageApplicationPrivileges;
-import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege;
-import org.elasticsearch.xpack.core.security.authz.privilege.Privilege;
-import org.elasticsearch.xpack.security.authz.AuthorizationService;
-
-import java.util.Collections;
-import java.util.Set;
-
-import static org.hamcrest.Matchers.containsInAnyOrder;
-import static org.hamcrest.Matchers.emptyIterable;
-import static org.hamcrest.Matchers.iterableWithSize;
-import static org.mockito.Mockito.mock;
-
-public class TransportGetUserPrivilegesActionTests extends ESTestCase {
-
-    public void testBuildResponseObject() {
-        final ManageApplicationPrivileges manageApplicationPrivileges = new ManageApplicationPrivileges(Sets.newHashSet("app01", "app02"));
-        final BytesArray query = new BytesArray("{\"term\":{\"public\":true}}");
-        final Role role = Role.builder("test", "role")
-            .cluster(Sets.newHashSet("monitor", "manage_watcher"), Collections.singleton(manageApplicationPrivileges))
-            .add(IndexPrivilege.get(Sets.newHashSet("read", "write")), "index-1")
-            .add(IndexPrivilege.ALL, "index-2", "index-3")
-            .add(
-                new FieldPermissions(new FieldPermissionsDefinition(new String[]{ "public.*" }, new String[0])),
-                Collections.singleton(query),
-                IndexPrivilege.READ, randomBoolean(), "index-4", "index-5")
-            .addApplicationPrivilege(new ApplicationPrivilege("app01", "read", "data:read"), Collections.singleton("*"))
-            .runAs(new Privilege(Sets.newHashSet("user01", "user02"), "user01", "user02"))
-            .build();
-
-        final TransportGetUserPrivilegesAction action = new TransportGetUserPrivilegesAction(mock(ThreadPool.class),
-            mock(TransportService.class), mock(ActionFilters.class), mock(AuthorizationService.class));
-        final GetUserPrivilegesResponse response = action.buildResponseObject(role);
-
-        assertThat(response.getClusterPrivileges(), containsInAnyOrder("monitor", "manage_watcher"));
-        assertThat(response.getConditionalClusterPrivileges(), containsInAnyOrder(manageApplicationPrivileges));
-
-        assertThat(response.getIndexPrivileges(), iterableWithSize(3));
-        final GetUserPrivilegesResponse.Indices index1 = findIndexPrivilege(response.getIndexPrivileges(), "index-1");
-        assertThat(index1.getIndices(), containsInAnyOrder("index-1"));
-        assertThat(index1.getPrivileges(), containsInAnyOrder("read", "write"));
-        assertThat(index1.getFieldSecurity(), emptyIterable());
-        assertThat(index1.getQueries(), emptyIterable());
-        final GetUserPrivilegesResponse.Indices index2 = findIndexPrivilege(response.getIndexPrivileges(), "index-2");
-        assertThat(index2.getIndices(), containsInAnyOrder("index-2", "index-3"));
-        assertThat(index2.getPrivileges(), containsInAnyOrder("all"));
-        assertThat(index2.getFieldSecurity(), emptyIterable());
-        assertThat(index2.getQueries(), emptyIterable());
-        final GetUserPrivilegesResponse.Indices index4 = findIndexPrivilege(response.getIndexPrivileges(), "index-4");
-        assertThat(index4.getIndices(), containsInAnyOrder("index-4", "index-5"));
-        assertThat(index4.getPrivileges(), containsInAnyOrder("read"));
-        assertThat(index4.getFieldSecurity(), containsInAnyOrder(
-            new FieldPermissionsDefinition.FieldGrantExcludeGroup(new String[]{ "public.*" }, new String[0])));
-        assertThat(index4.getQueries(), containsInAnyOrder(query));
-
-        assertThat(response.getApplicationPrivileges(), containsInAnyOrder(
-            RoleDescriptor.ApplicationResourcePrivileges.builder().application("app01").privileges("read").resources("*").build())
-        );
-
-        assertThat(response.getRunAs(), containsInAnyOrder("user01", "user02"));
-    }
-
-    private GetUserPrivilegesResponse.Indices findIndexPrivilege(Set<GetUserPrivilegesResponse.Indices> indices, String name) {
-        return indices.stream().filter(i -> i.getIndices().contains(name)).findFirst().get();
-    }
-}

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesActionTests.java

@@ -1,575 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.security.action.user;
-
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
-import org.elasticsearch.action.delete.DeleteAction;
-import org.elasticsearch.action.index.IndexAction;
-import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.PlainActionFuture;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.collect.MapBuilder;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.common.util.set.Sets;
-import org.elasticsearch.mock.orig.Mockito;
-import org.elasticsearch.tasks.Task;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.junit.annotations.TestLogging;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.Transport;
-import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesRequest;
-import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesResponse;
-import org.elasticsearch.xpack.core.security.authc.Authentication;
-import org.elasticsearch.xpack.core.security.authc.AuthenticationField;
-import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
-import org.elasticsearch.xpack.core.security.authz.permission.LimitedRole;
-import org.elasticsearch.xpack.core.security.authz.permission.ResourcePrivileges;
-import org.elasticsearch.xpack.core.security.authz.permission.Role;
-import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege;
-import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
-import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilege;
-import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege;
-import org.elasticsearch.xpack.core.security.user.User;
-import org.elasticsearch.xpack.security.authz.AuthorizationService;
-import org.elasticsearch.xpack.security.authz.store.NativePrivilegeStore;
-import org.hamcrest.Matchers;
-import org.junit.Before;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Set;
-
-import static java.util.Collections.emptyMap;
-import static org.elasticsearch.common.util.set.Sets.newHashSet;
-import static org.hamcrest.Matchers.arrayWithSize;
-import static org.hamcrest.Matchers.containsInAnyOrder;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.iterableWithSize;
-import static org.hamcrest.Matchers.notNullValue;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-@TestLogging("org.elasticsearch.xpack.security.action.user.TransportHasPrivilegesAction:TRACE," +
-        "org.elasticsearch.xpack.core.security.authz.permission.ApplicationPermission:DEBUG")
-public class TransportHasPrivilegesActionTests extends ESTestCase {
-
-    private User user;
-    private Role role;
-    private TransportHasPrivilegesAction action;
-    private List<ApplicationPrivilegeDescriptor> applicationPrivileges;
-
-    @Before
-    public void setup() {
-        user = new User(randomAlphaOfLengthBetween(4, 12));
-        final ThreadPool threadPool = mock(ThreadPool.class);
-        final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
-        final TransportService transportService = new TransportService(Settings.EMPTY, mock(Transport.class), null,
-            TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null, Collections.emptySet());
-
-        final Authentication authentication = mock(Authentication.class);
-        threadContext.putTransient(AuthenticationField.AUTHENTICATION_KEY, authentication);
-        when(threadPool.getThreadContext()).thenReturn(threadContext);
-
-        when(authentication.getUser()).thenReturn(user);
-
-        AuthorizationService authorizationService = mock(AuthorizationService.class);
-        Mockito.doAnswer(invocationOnMock -> {
-            ActionListener<Role> listener = (ActionListener<Role>) invocationOnMock.getArguments()[2];
-            listener.onResponse(role);
-            return null;
-        }).when(authorizationService).roles(eq(user), any(Authentication.class), any(ActionListener.class));
-
-        applicationPrivileges = new ArrayList<>();
-        NativePrivilegeStore privilegeStore = mock(NativePrivilegeStore.class);
-        Mockito.doAnswer(inv -> {
-            assertThat(inv.getArguments(), arrayWithSize(3));
-            ActionListener<List<ApplicationPrivilegeDescriptor>> listener
-                = (ActionListener<List<ApplicationPrivilegeDescriptor>>) inv.getArguments()[2];
-            logger.info("Privileges for ({}) are {}", Arrays.toString(inv.getArguments()), applicationPrivileges);
-            listener.onResponse(applicationPrivileges);
-            return null;
-        }).when(privilegeStore).getPrivileges(any(Collection.class), any(Collection.class), any(ActionListener.class));
-
-        action = new TransportHasPrivilegesAction(threadPool, transportService, mock(ActionFilters.class), authorizationService,
-            privilegeStore);
-    }
-
-    /**
-     * This tests that action names in the request are considered "matched" by the relevant named privilege
-     * (in this case that {@link DeleteAction} and {@link IndexAction} are satisfied by {@link IndexPrivilege#WRITE}).
-     */
-    public void testNamedIndexPrivilegesMatchApplicableActions() throws Exception {
-        role = Role.builder("test1")
-            .cluster(Collections.singleton("all"), Collections.emptyList())
-            .add(IndexPrivilege.WRITE, "academy")
-            .build();
-
-        final HasPrivilegesRequest request = new HasPrivilegesRequest();
-        request.username(user.principal());
-        request.clusterPrivileges(ClusterHealthAction.NAME);
-        request.indexPrivileges(RoleDescriptor.IndicesPrivileges.builder()
-                .indices("academy")
-                .privileges(DeleteAction.NAME, IndexAction.NAME)
-                .build());
-        request.applicationPrivileges(new RoleDescriptor.ApplicationResourcePrivileges[0]);
-        final PlainActionFuture<HasPrivilegesResponse> future = new PlainActionFuture<>();
-        action.doExecute(mock(Task.class), request, future);
-
-        final HasPrivilegesResponse response = future.get();
-        assertThat(response, notNullValue());
-        assertThat(response.getUsername(), is(user.principal()));
-        assertThat(response.isCompleteMatch(), is(true));
-
-        assertThat(response.getClusterPrivileges().size(), equalTo(1));
-        assertThat(response.getClusterPrivileges().get(ClusterHealthAction.NAME), equalTo(true));
-
-        assertThat(response.getIndexPrivileges(), Matchers.iterableWithSize(1));
-        final ResourcePrivileges result = response.getIndexPrivileges().iterator().next();
-        assertThat(result.getResource(), equalTo("academy"));
-        assertThat(result.getPrivileges().size(), equalTo(2));
-        assertThat(result.getPrivileges().get(DeleteAction.NAME), equalTo(true));
-        assertThat(result.getPrivileges().get(IndexAction.NAME), equalTo(true));
-    }
-
-    /**
-     * This tests that the action responds correctly when the user/role has some, but not all
-     * of the privileges being checked.
-     */
-    public void testMatchSubsetOfPrivileges() throws Exception {
-        role = Role.builder("test2")
-                .cluster(ClusterPrivilege.MONITOR)
-                .add(IndexPrivilege.INDEX, "academy")
-                .add(IndexPrivilege.WRITE, "initiative")
-                .build();
-
-        final HasPrivilegesRequest request = new HasPrivilegesRequest();
-        request.username(user.principal());
-        request.clusterPrivileges("monitor", "manage");
-        request.indexPrivileges(RoleDescriptor.IndicesPrivileges.builder()
-                .indices("academy", "initiative", "school")
-                .privileges("delete", "index", "manage")
-                .build());
-        request.applicationPrivileges(new RoleDescriptor.ApplicationResourcePrivileges[0]);
-        final PlainActionFuture<HasPrivilegesResponse> future = new PlainActionFuture<>();
-        action.doExecute(mock(Task.class), request, future);
-
-        final HasPrivilegesResponse response = future.get();
-        assertThat(response, notNullValue());
-        assertThat(response.getUsername(), is(user.principal()));
-        assertThat(response.isCompleteMatch(), is(false));
-        assertThat(response.getClusterPrivileges().size(), equalTo(2));
-        assertThat(response.getClusterPrivileges().get("monitor"), equalTo(true));
-        assertThat(response.getClusterPrivileges().get("manage"), equalTo(false));
-        assertThat(response.getIndexPrivileges(), Matchers.iterableWithSize(3));
-
-        final Iterator<ResourcePrivileges> indexPrivilegesIterator = response.getIndexPrivileges().iterator();
-        final ResourcePrivileges academy = indexPrivilegesIterator.next();
-        final ResourcePrivileges initiative = indexPrivilegesIterator.next();
-        final ResourcePrivileges school = indexPrivilegesIterator.next();
-
-        assertThat(academy.getResource(), equalTo("academy"));
-        assertThat(academy.getPrivileges().size(), equalTo(3));
-        assertThat(academy.getPrivileges().get("index"), equalTo(true)); // explicit
-        assertThat(academy.getPrivileges().get("delete"), equalTo(false));
-        assertThat(academy.getPrivileges().get("manage"), equalTo(false));
-
-        assertThat(initiative.getResource(), equalTo("initiative"));
-        assertThat(initiative.getPrivileges().size(), equalTo(3));
-        assertThat(initiative.getPrivileges().get("index"), equalTo(true)); // implied by write
-        assertThat(initiative.getPrivileges().get("delete"), equalTo(true)); // implied by write
-        assertThat(initiative.getPrivileges().get("manage"), equalTo(false));
-
-        assertThat(school.getResource(), equalTo("school"));
-        assertThat(school.getPrivileges().size(), equalTo(3));
-        assertThat(school.getPrivileges().get("index"), equalTo(false));
-        assertThat(school.getPrivileges().get("delete"), equalTo(false));
-        assertThat(school.getPrivileges().get("manage"), equalTo(false));
-    }
-
-    /**
-     * This tests that the action responds correctly when the user/role has none
-     * of the privileges being checked.
-     */
-    public void testMatchNothing() throws Exception {
-        role = Role.builder("test3")
-                .cluster(ClusterPrivilege.MONITOR)
-                .build();
-
-        final HasPrivilegesResponse response = hasPrivileges(RoleDescriptor.IndicesPrivileges.builder()
-                .indices("academy")
-                .privileges("read", "write")
-                .build(), Strings.EMPTY_ARRAY);
-        assertThat(response.getUsername(), is(user.principal()));
-        assertThat(response.isCompleteMatch(), is(false));
-        assertThat(response.getIndexPrivileges(), Matchers.iterableWithSize(1));
-        final ResourcePrivileges result = response.getIndexPrivileges().iterator().next();
-        assertThat(result.getResource(), equalTo("academy"));
-        assertThat(result.getPrivileges().size(), equalTo(2));
-        assertThat(result.getPrivileges().get("read"), equalTo(false));
-        assertThat(result.getPrivileges().get("write"), equalTo(false));
-    }
-
-    /**
-     * Wildcards in the request are treated as
-     * <em>does the user have ___ privilege on every possible index that matches this pattern?</em>
-     * Or, expressed differently,
-     * <em>does the user have ___ privilege on a wildcard that covers (is a superset of) this pattern?</em>
-     */
-    public void testWildcardHandling() throws Exception {
-        final ApplicationPrivilege kibanaRead = defineApplicationPrivilege("kibana", "read",
-                "data:read/*", "action:login", "action:view/dashboard");
-        final ApplicationPrivilege kibanaWrite = defineApplicationPrivilege("kibana", "write",
-                "data:write/*", "action:login", "action:view/dashboard");
-        final ApplicationPrivilege kibanaAdmin = defineApplicationPrivilege("kibana", "admin",
-                "action:login", "action:manage/*");
-        final ApplicationPrivilege kibanaViewSpace = defineApplicationPrivilege("kibana", "view-space",
-                "action:login", "space:view/*");
-        role = Role.builder("test3")
-                .add(IndexPrivilege.ALL, "logstash-*", "foo?")
-                .add(IndexPrivilege.READ, "abc*")
-                .add(IndexPrivilege.WRITE, "*xyz")
-                .addApplicationPrivilege(kibanaRead, Collections.singleton("*"))
-                .addApplicationPrivilege(kibanaViewSpace, newHashSet("space/engineering/*", "space/builds"))
-                .build();
-
-        final HasPrivilegesRequest request = new HasPrivilegesRequest();
-        request.username(user.principal());
-        request.clusterPrivileges(Strings.EMPTY_ARRAY);
-        request.indexPrivileges(
-                RoleDescriptor.IndicesPrivileges.builder()
-                        .indices("logstash-2016-*")
-                        .privileges("write") // Yes, because (ALL,"logstash-*")
-                        .build(),
-                RoleDescriptor.IndicesPrivileges.builder()
-                        .indices("logstash-*")
-                        .privileges("read") // Yes, because (ALL,"logstash-*")
-                        .build(),
-                RoleDescriptor.IndicesPrivileges.builder()
-                        .indices("log*")
-                        .privileges("manage") // No, because "log*" includes indices that "logstash-*" does not
-                        .build(),
-                RoleDescriptor.IndicesPrivileges.builder()
-                        .indices("foo*", "foo?")
-                        .privileges("read") // Yes, "foo?", but not "foo*", because "foo*" > "foo?"
-                        .build(),
-                RoleDescriptor.IndicesPrivileges.builder()
-                        .indices("abcd*")
-                        .privileges("read", "write") // read = Yes, because (READ, "abc*"), write = No
-                        .build(),
-                RoleDescriptor.IndicesPrivileges.builder()
-                        .indices("abc*xyz")
-                        .privileges("read", "write", "manage") // read = Yes ( READ "abc*"), write = Yes (WRITE, "*xyz"), manage = No
-                        .build(),
-                RoleDescriptor.IndicesPrivileges.builder()
-                        .indices("a*xyz")
-                        .privileges("read", "write", "manage") // read = No, write = Yes (WRITE, "*xyz"), manage = No
-                        .build()
-        );
-
-        request.applicationPrivileges(
-                RoleDescriptor.ApplicationResourcePrivileges.builder()
-                        .resources("*")
-                        .application("kibana")
-                        .privileges(Sets.union(kibanaRead.name(), kibanaWrite.name())) // read = Yes, write = No
-                        .build(),
-                RoleDescriptor.ApplicationResourcePrivileges.builder()
-                        .resources("space/engineering/project-*", "space/*") // project-* = Yes, space/* = Not
-                        .application("kibana")
-                        .privileges("space:view/dashboard")
-                        .build()
-        );
-
-        final PlainActionFuture<HasPrivilegesResponse> future = new PlainActionFuture<>();
-        action.doExecute(mock(Task.class), request, future);
-
-        final HasPrivilegesResponse response = future.get();
-        assertThat(response, notNullValue());
-        assertThat(response.getUsername(), is(user.principal()));
-        assertThat(response.isCompleteMatch(), is(false));
-        assertThat(response.getIndexPrivileges(), Matchers.iterableWithSize(8));
-        assertThat(response.getIndexPrivileges(),
-                containsInAnyOrder(
-                        ResourcePrivileges.builder("logstash-2016-*").addPrivileges(Collections.singletonMap("write", true)).build(),
-                        ResourcePrivileges.builder("logstash-*").addPrivileges(Collections.singletonMap("read", true)).build(),
-                        ResourcePrivileges.builder("log*").addPrivileges(Collections.singletonMap("manage", false)).build(),
-                        ResourcePrivileges.builder("foo?").addPrivileges(Collections.singletonMap("read", true)).build(),
-                        ResourcePrivileges.builder("foo*").addPrivileges(Collections.singletonMap("read", false)).build(),
-                        ResourcePrivileges.builder("abcd*").addPrivileges(mapBuilder().put("read", true).put("write", false).map()).build(),
-                        ResourcePrivileges.builder("abc*xyz")
-                                .addPrivileges(mapBuilder().put("read", true).put("write", true).put("manage", false).map()).build(),
-                        ResourcePrivileges.builder("a*xyz")
-                                .addPrivileges(mapBuilder().put("read", false).put("write", true).put("manage", false).map()).build()));
-        assertThat(response.getApplicationPrivileges().entrySet(), Matchers.iterableWithSize(1));
-        final Set<ResourcePrivileges> kibanaPrivileges = response.getApplicationPrivileges().get("kibana");
-        assertThat(kibanaPrivileges, Matchers.iterableWithSize(3));
-        assertThat(Strings.collectionToCommaDelimitedString(kibanaPrivileges), kibanaPrivileges, containsInAnyOrder(
-                ResourcePrivileges.builder("*").addPrivileges(mapBuilder().put("read", true).put("write", false).map()).build(),
-                ResourcePrivileges.builder("space/engineering/project-*")
-                        .addPrivileges(Collections.singletonMap("space:view/dashboard", true)).build(),
-                ResourcePrivileges.builder("space/*").addPrivileges(Collections.singletonMap("space:view/dashboard", false)).build()));
-    }
-
-    private ApplicationPrivilege defineApplicationPrivilege(String app, String name, String ... actions) {
-        this.applicationPrivileges.add(new ApplicationPrivilegeDescriptor(app, name, newHashSet(actions), emptyMap()));
-        return new ApplicationPrivilege(app, name, actions);
-    }
-
-    public void testCheckingIndexPermissionsDefinedOnDifferentPatterns() throws Exception {
-        role = Role.builder("test-write")
-                .add(IndexPrivilege.INDEX, "apache-*")
-                .add(IndexPrivilege.DELETE, "apache-2016-*")
-                .build();
-
-        final HasPrivilegesResponse response = hasPrivileges(RoleDescriptor.IndicesPrivileges.builder()
-                .indices("apache-2016-12", "apache-2017-01")
-                .privileges("index", "delete")
-                .build(), Strings.EMPTY_ARRAY);
-        assertThat(response.isCompleteMatch(), is(false));
-        assertThat(response.getIndexPrivileges(), Matchers.iterableWithSize(2));
-        assertThat(response.getIndexPrivileges(), containsInAnyOrder(
-                ResourcePrivileges.builder("apache-2016-12").addPrivileges(
-                        MapBuilder.newMapBuilder(new LinkedHashMap<String, Boolean>())
-                                .put("index", true).put("delete", true).map()).build(),
-                ResourcePrivileges.builder("apache-2017-01").addPrivileges(
-                        MapBuilder.newMapBuilder(new LinkedHashMap<String, Boolean>())
-                                .put("index", true).put("delete", false).map()).build()
-        ));
-    }
-
-    public void testCheckingApplicationPrivilegesOnDifferentApplicationsAndResources() throws Exception {
-        final ApplicationPrivilege app1Read = defineApplicationPrivilege("app1", "read", "data:read/*");
-        final ApplicationPrivilege app1Write = defineApplicationPrivilege("app1", "write", "data:write/*");
-        final ApplicationPrivilege app1All = defineApplicationPrivilege("app1", "all", "*");
-        final ApplicationPrivilege app2Read = defineApplicationPrivilege("app2", "read", "data:read/*");
-        final ApplicationPrivilege app2Write = defineApplicationPrivilege("app2", "write", "data:write/*");
-        final ApplicationPrivilege app2All = defineApplicationPrivilege("app2", "all", "*");
-
-        role = Role.builder("test-role")
-                .addApplicationPrivilege(app1Read, Collections.singleton("foo/*"))
-                .addApplicationPrivilege(app1All, Collections.singleton("foo/bar/baz"))
-                .addApplicationPrivilege(app2Read, Collections.singleton("foo/bar/*"))
-                .addApplicationPrivilege(app2Write, Collections.singleton("*/bar/*"))
-                .build();
-
-        final HasPrivilegesResponse response = hasPrivileges(new RoleDescriptor.IndicesPrivileges[0],
-                new RoleDescriptor.ApplicationResourcePrivileges[]{
-                        RoleDescriptor.ApplicationResourcePrivileges.builder()
-                                .application("app1")
-                                .resources("foo/1", "foo/bar/2", "foo/bar/baz", "baz/bar/foo")
-                                .privileges("read", "write", "all")
-                                .build(),
-                        RoleDescriptor.ApplicationResourcePrivileges.builder()
-                                .application("app2")
-                                .resources("foo/1", "foo/bar/2", "foo/bar/baz", "baz/bar/foo")
-                                .privileges("read", "write", "all")
-                                .build()
-                }, Strings.EMPTY_ARRAY);
-
-        assertThat(response.isCompleteMatch(), is(false));
-        assertThat(response.getIndexPrivileges(), Matchers.emptyIterable());
-        assertThat(response.getApplicationPrivileges().entrySet(), Matchers.iterableWithSize(2));
-        final Set<ResourcePrivileges> app1 = response.getApplicationPrivileges().get("app1");
-        assertThat(app1, Matchers.iterableWithSize(4));
-        assertThat(Strings.collectionToCommaDelimitedString(app1), app1, containsInAnyOrder(
-                ResourcePrivileges.builder("foo/1")
-                        .addPrivileges(MapBuilder.newMapBuilder(new LinkedHashMap<String, Boolean>()).put("read", true).put("write", false)
-                                .put("all", false).map())
-                        .build(),
-                ResourcePrivileges.builder("foo/bar/2")
-                        .addPrivileges(MapBuilder.newMapBuilder(new LinkedHashMap<String, Boolean>()).put("read", true).put("write", false)
-                                .put("all", false).map())
-                        .build(),
-                ResourcePrivileges.builder("foo/bar/baz")
-                        .addPrivileges(MapBuilder.newMapBuilder(new LinkedHashMap<String, Boolean>()).put("read", true).put("write", true)
-                                .put("all", true).map())
-                        .build(),
-                ResourcePrivileges.builder("baz/bar/foo").addPrivileges(MapBuilder.newMapBuilder(new LinkedHashMap<String, Boolean>())
-                        .put("read", false).put("write", false).put("all", false).map()).build()));
-        final Set<ResourcePrivileges> app2 = response.getApplicationPrivileges().get("app2");
-        assertThat(app2, Matchers.iterableWithSize(4));
-        assertThat(Strings.collectionToCommaDelimitedString(app2), app2, containsInAnyOrder(
-                ResourcePrivileges.builder("foo/1")
-                        .addPrivileges(MapBuilder.newMapBuilder(new LinkedHashMap<String, Boolean>()).put("read", false).put("write", false)
-                                .put("all", false).map())
-                        .build(),
-                ResourcePrivileges.builder("foo/bar/2")
-                        .addPrivileges(MapBuilder.newMapBuilder(new LinkedHashMap<String, Boolean>()).put("read", true).put("write", true)
-                                .put("all", false).map())
-                        .build(),
-                ResourcePrivileges.builder("foo/bar/baz")
-                        .addPrivileges(MapBuilder.newMapBuilder(new LinkedHashMap<String, Boolean>()).put("read", true).put("write", true)
-                                .put("all", false).map())
-                        .build(),
-                ResourcePrivileges.builder("baz/bar/foo").addPrivileges(MapBuilder.newMapBuilder(new LinkedHashMap<String, Boolean>())
-                        .put("read", false).put("write", true).put("all", false).map()).build()
-        ));
-    }
-
-    public void testCheckingApplicationPrivilegesWithComplexNames() throws Exception {
-        final String appName = randomAlphaOfLength(1).toLowerCase(Locale.ROOT) + randomAlphaOfLengthBetween(3, 10);
-        final String action1 = randomAlphaOfLength(1).toLowerCase(Locale.ROOT) + randomAlphaOfLengthBetween(2, 5);
-        final String action2 = randomAlphaOfLength(1).toLowerCase(Locale.ROOT) + randomAlphaOfLengthBetween(6, 9);
-
-        final ApplicationPrivilege priv1 = defineApplicationPrivilege(appName, action1, "DATA:read/*", "ACTION:" + action1);
-
-        role = Role.builder("test-write")
-            .addApplicationPrivilege(priv1, Collections.singleton("user/*/name"))
-            .build();
-
-        final HasPrivilegesResponse response = hasPrivileges(
-            new RoleDescriptor.IndicesPrivileges[0],
-            new RoleDescriptor.ApplicationResourcePrivileges[]{
-                RoleDescriptor.ApplicationResourcePrivileges.builder()
-                    .application(appName)
-                    .resources("user/hawkeye/name")
-                    .privileges("DATA:read/user/*", "ACTION:" + action1, "ACTION:" + action2, action1, action2)
-                    .build()
-            },
-            "monitor");
-        assertThat(response.isCompleteMatch(), is(false));
-        assertThat(response.getApplicationPrivileges().keySet(), containsInAnyOrder(appName));
-        assertThat(response.getApplicationPrivileges().get(appName), iterableWithSize(1));
-        assertThat(response.getApplicationPrivileges().get(appName), containsInAnyOrder(
-            ResourcePrivileges.builder("user/hawkeye/name").addPrivileges(MapBuilder.newMapBuilder(new LinkedHashMap<String, Boolean>())
-                .put("DATA:read/user/*", true)
-                .put("ACTION:" + action1, true)
-                .put("ACTION:" + action2, false)
-                .put(action1, true)
-                .put(action2, false)
-                .map()).build()
-        ));
-    }
-
-    public void testIsCompleteMatch() throws Exception {
-        final ApplicationPrivilege kibanaRead = defineApplicationPrivilege("kibana", "read", "data:read/*");
-        final ApplicationPrivilege kibanaWrite = defineApplicationPrivilege("kibana", "write", "data:write/*");
-        role = Role.builder("test-write")
-                .cluster(ClusterPrivilege.MONITOR)
-                .add(IndexPrivilege.READ, "read-*")
-                .add(IndexPrivilege.ALL, "all-*")
-                .addApplicationPrivilege(kibanaRead, Collections.singleton("*"))
-                .build();
-
-        assertThat(hasPrivileges(indexPrivileges("read", "read-123", "read-456", "all-999"), "monitor").isCompleteMatch(), is(true));
-        assertThat(hasPrivileges(indexPrivileges("read", "read-123", "read-456", "all-999"), "manage").isCompleteMatch(), is(false));
-        assertThat(hasPrivileges(indexPrivileges("write", "read-123", "read-456", "all-999"), "monitor").isCompleteMatch(), is(false));
-        assertThat(hasPrivileges(indexPrivileges("write", "read-123", "read-456", "all-999"), "manage").isCompleteMatch(), is(false));
-        assertThat(hasPrivileges(
-                new RoleDescriptor.IndicesPrivileges[]{
-                        RoleDescriptor.IndicesPrivileges.builder()
-                                .indices("read-a")
-                                .privileges("read")
-                                .build(),
-                        RoleDescriptor.IndicesPrivileges.builder()
-                                .indices("all-b")
-                                .privileges("read", "write")
-                                .build()
-                },
-                new RoleDescriptor.ApplicationResourcePrivileges[]{
-                        RoleDescriptor.ApplicationResourcePrivileges.builder()
-                                .application("kibana")
-                                .resources("*")
-                                .privileges("read")
-                                .build()
-                },
-                "monitor").isCompleteMatch(), is(true));
-        assertThat(hasPrivileges(
-                new RoleDescriptor.IndicesPrivileges[]{indexPrivileges("read", "read-123", "read-456", "all-999")},
-                new RoleDescriptor.ApplicationResourcePrivileges[]{
-                        RoleDescriptor.ApplicationResourcePrivileges.builder()
-                                .application("kibana").resources("*").privileges("read").build(),
-                        RoleDescriptor.ApplicationResourcePrivileges.builder()
-                                .application("kibana").resources("*").privileges("write").build()
-                },
-                "monitor").isCompleteMatch(), is(false));
-    }
-
-    public void testLimitedRoleHasPrivilegesApi() throws Exception {
-        final ApplicationPrivilege kibanaRead = defineApplicationPrivilege("kibana", "read", "data:read/*");
-        final ApplicationPrivilege kibanaWrite = defineApplicationPrivilege("kibana", "write", "data:write/*");
-        Role baseRole = Role.builder("base-role").cluster(Sets.newHashSet("manage", "monitor"), Collections.emptySet())
-                .add(IndexPrivilege.ALL, "all-*").addApplicationPrivilege(kibanaRead, Collections.singleton("*"))
-                .addApplicationPrivilege(kibanaWrite, Collections.singleton("*")).build();
-
-        Role limitedByRole = Role.builder("limited-by").cluster(ClusterPrivilege.MONITOR).add(IndexPrivilege.READ, "all-read-*")
-                .addApplicationPrivilege(kibanaRead, Collections.singleton("*")).build();
-
-        role = LimitedRole.createLimitedRole(baseRole, limitedByRole);
-
-        assertThat(hasPrivileges(indexPrivileges("read", "all-read-1", "all-read-2", "all-read-*"), "monitor").isCompleteMatch(), is(true));
-        assertThat(hasPrivileges(indexPrivileges("read", "all-1", "all-999"), "monitor").isCompleteMatch(), is(false));
-        assertThat(hasPrivileges(indexPrivileges("read", "all-999"), "manage").isCompleteMatch(), is(false));
-        assertThat(hasPrivileges(indexPrivileges("write", "all-999"), "monitor").isCompleteMatch(), is(false));
-        assertThat(hasPrivileges(indexPrivileges("write", "all-*"), "manage").isCompleteMatch(), is(false));
-
-        HasPrivilegesResponse response = hasPrivileges(indexPrivileges("read", "all-read-999"), "manage", "monitor");
-        assertThat(response.getClusterPrivileges().get("manage"), is(false));
-        assertThat(response.getClusterPrivileges().get("monitor"), is(true));
-        assertThat(response.getIndexPrivileges(), Matchers.iterableWithSize(1));
-        assertThat(response.getIndexPrivileges(), containsInAnyOrder(ResourcePrivileges.builder("all-read-999")
-                .addPrivileges(MapBuilder.newMapBuilder(new LinkedHashMap<String, Boolean>()).put("read", true).map()).build()));
-
-        response = hasPrivileges(new RoleDescriptor.IndicesPrivileges[] { indexPrivileges("read", "all-read-*") },
-                new RoleDescriptor.ApplicationResourcePrivileges[] {
-                        RoleDescriptor.ApplicationResourcePrivileges.builder().application("kibana").resources("*").privileges("read")
-                                .build(),
-                        RoleDescriptor.ApplicationResourcePrivileges.builder().application("kibana").resources("*").privileges("write")
-                                .build() },
-                "monitor");
-        assertThat(response.isCompleteMatch(), is(false));
-        final Set<ResourcePrivileges> kibanaPrivileges = response.getApplicationPrivileges().get("kibana");
-        assertThat(kibanaPrivileges, containsInAnyOrder(
-                ResourcePrivileges.builder("*").addPrivileges(mapBuilder().put("write", false).put("read", true).map()).build()));
-    }
-
-    private RoleDescriptor.IndicesPrivileges indexPrivileges(String priv, String... indices) {
-        return RoleDescriptor.IndicesPrivileges.builder()
-                .indices(indices)
-                .privileges(priv)
-                .build();
-    }
-
-    private HasPrivilegesResponse hasPrivileges(RoleDescriptor.IndicesPrivileges indicesPrivileges, String... clusterPrivileges)
-            throws Exception {
-        return hasPrivileges(
-                new RoleDescriptor.IndicesPrivileges[]{indicesPrivileges},
-                new RoleDescriptor.ApplicationResourcePrivileges[0],
-                clusterPrivileges
-        );
-    }
-
-    private HasPrivilegesResponse hasPrivileges(RoleDescriptor.IndicesPrivileges[] indicesPrivileges,
-                                                RoleDescriptor.ApplicationResourcePrivileges[] appPrivileges,
-                                                String... clusterPrivileges) throws Exception {
-        final HasPrivilegesRequest request = new HasPrivilegesRequest();
-        request.username(user.principal());
-        request.clusterPrivileges(clusterPrivileges);
-        request.indexPrivileges(indicesPrivileges);
-        request.applicationPrivileges(appPrivileges);
-        final PlainActionFuture<HasPrivilegesResponse> future = new PlainActionFuture<>();
-        action.doExecute(mock(Task.class), request, future);
-        final HasPrivilegesResponse response = future.get();
-        assertThat(response, notNullValue());
-        return response;
-    }
-
-    private static MapBuilder<String, Boolean> mapBuilder() {
-        return MapBuilder.newMapBuilder();
-    }
-
-}

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/AuditTrailServiceTests.java

@@ -13,15 +13,18 @@ import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationToken;
 import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationInfo;
 import org.elasticsearch.xpack.security.transport.filter.IPFilter;
 import org.elasticsearch.xpack.security.transport.filter.SecurityIpFilterRule;
 import org.junit.Before;
 
 import java.net.InetAddress;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 
 import static java.util.Collections.unmodifiableList;
+import static org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail.PRINCIPAL_ROLES_FIELD_NAME;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyZeroInteractions;
@@ -147,13 +150,14 @@ public class AuditTrailServiceTests extends ESTestCase {
     public void testAccessGranted() throws Exception {
         Authentication authentication =new Authentication(new User("_username", "r1"), new RealmRef(null, null, null),
                 new RealmRef(null, null, null));
-        String[] roles = new String[] { randomAlphaOfLengthBetween(1, 6) };
+        AuthorizationInfo authzInfo =
+            () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, new String[] { randomAlphaOfLengthBetween(1, 6) });
         final String requestId = randomAlphaOfLengthBetween(6, 12);
-        service.accessGranted(requestId, authentication, "_action", message, roles);
+        service.accessGranted(requestId, authentication, "_action", message, authzInfo);
         verify(licenseState).isAuditingAllowed();
         if (isAuditingAllowed) {
             for (AuditTrail auditTrail : auditTrails) {
-                verify(auditTrail).accessGranted(requestId, authentication, "_action", message, roles);
+                verify(auditTrail).accessGranted(requestId, authentication, "_action", message, authzInfo);
             }
         } else {
             verifyZeroInteractions(auditTrails.toArray((Object[]) new AuditTrail[auditTrails.size()]));
@@ -163,13 +167,14 @@ public class AuditTrailServiceTests extends ESTestCase {
     public void testAccessDenied() throws Exception {
         Authentication authentication = new Authentication(new User("_username", "r1"), new RealmRef(null, null, null),
                 new RealmRef(null, null, null));
-        String[] roles = new String[] { randomAlphaOfLengthBetween(1, 6) };
+        AuthorizationInfo authzInfo =
+            () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, new String[] { randomAlphaOfLengthBetween(1, 6) });
         final String requestId = randomAlphaOfLengthBetween(6, 12);
-        service.accessDenied(requestId, authentication, "_action", message, roles);
+        service.accessDenied(requestId, authentication, "_action", message, authzInfo);
         verify(licenseState).isAuditingAllowed();
         if (isAuditingAllowed) {
             for (AuditTrail auditTrail : auditTrails) {
-                verify(auditTrail).accessDenied(requestId, authentication, "_action", message, roles);
+                verify(auditTrail).accessDenied(requestId, authentication, "_action", message, authzInfo);
             }
         } else {
             verifyZeroInteractions(auditTrails.toArray((Object[]) new AuditTrail[auditTrails.size()]));

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailFilterTests.java

@@ -32,6 +32,7 @@ import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail.AuditEventMetaInfo;
 import org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrailTests.MockMessage;
 import org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrailTests.RestContent;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationInfo;
 import org.elasticsearch.xpack.security.rest.RemoteHostHeader;
 import org.elasticsearch.xpack.security.transport.filter.SecurityIpFilterRule;
 import org.junit.Before;
@@ -47,6 +48,7 @@ import java.util.List;
 import java.util.Optional;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail.PRINCIPAL_ROLES_FIELD_NAME;
 import static org.hamcrest.Matchers.is;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
@@ -131,7 +133,8 @@ public class LoggingAuditTrailFilterTests extends ESTestCase {
         // role field matches
         assertTrue("Matches the role filter predicate.", auditTrail.eventFilterPolicyRegistry.ignorePredicate()
                 .test(new AuditEventMetaInfo(Optional.empty(), Optional.empty(),
-                        Optional.of(randomSubsetOf(randomIntBetween(1, filteredRoles.size()), filteredRoles).toArray(new String[0])),
+                        Optional.of(authzInfo(
+                            randomSubsetOf(randomIntBetween(1, filteredRoles.size()), filteredRoles).toArray(new String[0]))),
                         Optional.empty())));
         final List<String> unfilteredRoles = new ArrayList<>();
         unfilteredRoles.add(null);
@@ -139,7 +142,7 @@ public class LoggingAuditTrailFilterTests extends ESTestCase {
         // null role among roles field does NOT match
         assertFalse("Does not match the role filter predicate because of null role.",
                 auditTrail.eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.empty(), Optional.empty(),
-                        Optional.of(unfilteredRoles.toArray(new String[0])), Optional.empty())));
+                        Optional.of(authzInfo(unfilteredRoles.toArray(new String[0]))), Optional.empty())));
         // indices field matches
         assertTrue("Matches the index filter predicate.",
                 auditTrail.eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.empty(), Optional.empty(),
@@ -185,7 +188,7 @@ public class LoggingAuditTrailFilterTests extends ESTestCase {
         assertTrue("Matches the filter predicate.", auditTrail.eventFilterPolicyRegistry.ignorePredicate()
                 .test(new AuditEventMetaInfo(
                 Optional.of(randomFrom(filteredUsers)), Optional.of(randomFrom(filteredRealms)),
-                Optional.of(randomSubsetOf(randomIntBetween(1, filteredRoles.size()), filteredRoles).toArray(new String[0])),
+                Optional.of(authzInfo(randomSubsetOf(randomIntBetween(1, filteredRoles.size()), filteredRoles).toArray(new String[0]))),
                 Optional.of(randomSubsetOf(randomIntBetween(1, filteredIndices.size()), filteredIndices).toArray(new String[0])))));
         final User unfilteredUser;
         if (randomBoolean()) {
@@ -198,22 +201,26 @@ public class LoggingAuditTrailFilterTests extends ESTestCase {
         assertFalse("Does not match the filter predicate because of the user.",
                 auditTrail.eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.of(unfilteredUser),
                         Optional.of(randomFrom(filteredRealms)),
-                        Optional.of(randomSubsetOf(randomIntBetween(1, filteredRoles.size()), filteredRoles).toArray(new String[0])),
+                        Optional.of(authzInfo(randomSubsetOf(randomIntBetween(1, filteredRoles.size()), filteredRoles)
+                            .toArray(new String[0]))),
                         Optional.of(randomSubsetOf(randomIntBetween(1, filteredIndices.size()), filteredIndices).toArray(new String[0])))));
         assertFalse("Does not match the filter predicate because of the empty user.",
                 auditTrail.eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.empty(),
                         Optional.of(randomFrom(filteredRealms)),
-                        Optional.of(randomSubsetOf(randomIntBetween(1, filteredRoles.size()), filteredRoles).toArray(new String[0])),
+                        Optional.of(authzInfo(randomSubsetOf(randomIntBetween(1, filteredRoles.size()), filteredRoles)
+                            .toArray(new String[0]))),
                         Optional.of(randomSubsetOf(randomIntBetween(1, filteredIndices.size()), filteredIndices).toArray(new String[0])))));
         assertFalse("Does not match the filter predicate because of the realm.",
                 auditTrail.eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.of(randomFrom(filteredUsers)),
                         Optional.of(UNFILTER_MARKER + randomAlphaOfLengthBetween(1, 8)),
-                        Optional.of(randomSubsetOf(randomIntBetween(1, filteredRoles.size()), filteredRoles).toArray(new String[0])),
+                        Optional.of(authzInfo(randomSubsetOf(randomIntBetween(1, filteredRoles.size()), filteredRoles)
+                            .toArray(new String[0]))),
                         Optional.of(randomSubsetOf(randomIntBetween(1, filteredIndices.size()), filteredIndices).toArray(new String[0])))));
         assertFalse("Does not match the filter predicate because of the empty realm.",
                 auditTrail.eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.of(randomFrom(filteredUsers)),
                         Optional.empty(),
-                        Optional.of(randomSubsetOf(randomIntBetween(1, filteredRoles.size()), filteredRoles).toArray(new String[0])),
+                        Optional.of(authzInfo(randomSubsetOf(randomIntBetween(1, filteredRoles.size()), filteredRoles)
+                                .toArray(new String[0]))),
                         Optional.of(randomSubsetOf(randomIntBetween(1, filteredIndices.size()), filteredIndices).toArray(new String[0])))));
         final List<String> someRolesDoNotMatch = new ArrayList<>(randomSubsetOf(randomIntBetween(0, filteredRoles.size()), filteredRoles));
         for (int i = 0; i < randomIntBetween(1, 8); i++) {
@@ -221,9 +228,9 @@ public class LoggingAuditTrailFilterTests extends ESTestCase {
         }
         assertFalse("Does not match the filter predicate because of some of the roles.",
                 auditTrail.eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.of(randomFrom(filteredUsers)),
-                        Optional.of(randomFrom(filteredRealms)), Optional.of(someRolesDoNotMatch.toArray(new String[0])),
+                        Optional.of(randomFrom(filteredRealms)), Optional.of(authzInfo(someRolesDoNotMatch.toArray(new String[0]))),
                         Optional.of(randomSubsetOf(randomIntBetween(1, filteredIndices.size()), filteredIndices).toArray(new String[0])))));
-        final Optional<String[]> emptyRoles = randomBoolean() ? Optional.empty() : Optional.of(new String[0]);
+        final Optional<AuthorizationInfo> emptyRoles = randomBoolean() ? Optional.empty() : Optional.of(authzInfo(new String[0]));
         assertFalse("Does not match the filter predicate because of the empty roles.",
                 auditTrail.eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.of(randomFrom(filteredUsers)),
                         Optional.of(randomFrom(filteredRealms)), emptyRoles,
@@ -236,13 +243,15 @@ public class LoggingAuditTrailFilterTests extends ESTestCase {
         assertFalse("Does not match the filter predicate because of some of the indices.",
                 auditTrail.eventFilterPolicyRegistry.ignorePredicate()
                 .test(new AuditEventMetaInfo(Optional.of(randomFrom(filteredUsers)), Optional.of(randomFrom(filteredRealms)),
-                        Optional.of(randomSubsetOf(randomIntBetween(1, filteredRoles.size()), filteredRoles).toArray(new String[0])),
+                        Optional.of(authzInfo(randomSubsetOf(randomIntBetween(1, filteredRoles.size()), filteredRoles)
+                                .toArray(new String[0]))),
                         Optional.of(someIndicesDoNotMatch.toArray(new String[0])))));
         final Optional<String[]> emptyIndices = randomBoolean() ? Optional.empty() : Optional.of(new String[0]);
         assertFalse("Does not match the filter predicate because of the empty indices.",
                 auditTrail.eventFilterPolicyRegistry.ignorePredicate()
                 .test(new AuditEventMetaInfo(Optional.of(randomFrom(filteredUsers)), Optional.of(randomFrom(filteredRealms)),
-                        Optional.of(randomSubsetOf(randomIntBetween(1, filteredRoles.size()), filteredRoles).toArray(new String[0])),
+                        Optional.of(authzInfo(randomSubsetOf(randomIntBetween(1, filteredRoles.size()), filteredRoles)
+                            .toArray(new String[0]))),
                         emptyIndices)));
     }
 
@@ -284,7 +293,8 @@ public class LoggingAuditTrailFilterTests extends ESTestCase {
         assertTrue("Matches the filter predicate.",
                 auditTrail.eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.of(randomFrom(filteredUsers)),
                         Optional.of(randomFrom(filteredRealms)),
-                        Optional.of(randomSubsetOf(randomIntBetween(1, filteredRoles.size()), filteredRoles).toArray(new String[0])),
+                        Optional.of(authzInfo(randomSubsetOf(randomIntBetween(1, filteredRoles.size()), filteredRoles)
+                            .toArray(new String[0]))),
                         Optional.of(randomSubsetOf(randomIntBetween(1, filteredIndices.size()), filteredIndices).toArray(new String[0])))));
         final User unfilteredUser;
         if (randomBoolean()) {
@@ -297,22 +307,26 @@ public class LoggingAuditTrailFilterTests extends ESTestCase {
         assertFalse("Does not match the filter predicate because of the user.",
                 auditTrail.eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.of(unfilteredUser),
                         Optional.of(randomFrom(filteredRealms)),
-                        Optional.of(randomSubsetOf(randomIntBetween(1, filteredRoles.size()), filteredRoles).toArray(new String[0])),
+                        Optional.of(authzInfo(randomSubsetOf(randomIntBetween(1, filteredRoles.size()), filteredRoles)
+                            .toArray(new String[0]))),
                         Optional.of(randomSubsetOf(randomIntBetween(1, filteredIndices.size()), filteredIndices).toArray(new String[0])))));
         assertTrue("Matches the filter predicate because of the empty user.",
                 auditTrail.eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.empty(),
                         Optional.of(randomFrom(filteredRealms)),
-                        Optional.of(randomSubsetOf(randomIntBetween(1, filteredRoles.size()), filteredRoles).toArray(new String[0])),
+                        Optional.of(authzInfo(randomSubsetOf(randomIntBetween(1, filteredRoles.size()), filteredRoles)
+                            .toArray(new String[0]))),
                         Optional.of(randomSubsetOf(randomIntBetween(1, filteredIndices.size()), filteredIndices).toArray(new String[0])))));
         assertFalse("Does not match the filter predicate because of the realm.",
                 auditTrail.eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.of(randomFrom(filteredUsers)),
                         Optional.of(UNFILTER_MARKER + randomAlphaOfLengthBetween(1, 8)),
-                        Optional.of(randomSubsetOf(randomIntBetween(1, filteredRoles.size()), filteredRoles).toArray(new String[0])),
+                        Optional.of(authzInfo(randomSubsetOf(randomIntBetween(1, filteredRoles.size()), filteredRoles)
+                            .toArray(new String[0]))),
                         Optional.of(randomSubsetOf(randomIntBetween(1, filteredIndices.size()), filteredIndices).toArray(new String[0])))));
         assertTrue("Matches the filter predicate because of the empty realm.",
                 auditTrail.eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.of(randomFrom(filteredUsers)),
                         Optional.empty(),
-                        Optional.of(randomSubsetOf(randomIntBetween(1, filteredRoles.size()), filteredRoles).toArray(new String[0])),
+                        Optional.of(authzInfo(randomSubsetOf(randomIntBetween(1, filteredRoles.size()), filteredRoles)
+                            .toArray(new String[0]))),
                         Optional.of(randomSubsetOf(randomIntBetween(1, filteredIndices.size()), filteredIndices).toArray(new String[0])))));
         final List<String> someRolesDoNotMatch = new ArrayList<>(randomSubsetOf(randomIntBetween(0, filteredRoles.size()), filteredRoles));
         for (int i = 0; i < randomIntBetween(1, 8); i++) {
@@ -320,9 +334,9 @@ public class LoggingAuditTrailFilterTests extends ESTestCase {
         }
         assertFalse("Does not match the filter predicate because of some of the roles.",
                 auditTrail.eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.of(randomFrom(filteredUsers)),
-                        Optional.of(randomFrom(filteredRealms)), Optional.of(someRolesDoNotMatch.toArray(new String[0])),
+                        Optional.of(randomFrom(filteredRealms)), Optional.of(authzInfo(someRolesDoNotMatch.toArray(new String[0]))),
                         Optional.of(randomSubsetOf(randomIntBetween(1, filteredIndices.size()), filteredIndices).toArray(new String[0])))));
-        final Optional<String[]> emptyRoles = randomBoolean() ? Optional.empty() : Optional.of(new String[0]);
+        final Optional<AuthorizationInfo> emptyRoles = randomBoolean() ? Optional.empty() : Optional.of(authzInfo(new String[0]));
         assertTrue("Matches the filter predicate because of the empty roles.",
                 auditTrail.eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.of(randomFrom(filteredUsers)),
                         Optional.of(randomFrom(filteredRealms)), emptyRoles,
@@ -335,19 +349,23 @@ public class LoggingAuditTrailFilterTests extends ESTestCase {
         assertFalse("Does not match the filter predicate because of some of the indices.",
                 auditTrail.eventFilterPolicyRegistry.ignorePredicate()
                 .test(new AuditEventMetaInfo(Optional.of(randomFrom(filteredUsers)), Optional.of(randomFrom(filteredRealms)),
-                        Optional.of(randomSubsetOf(randomIntBetween(1, filteredRoles.size()), filteredRoles).toArray(new String[0])),
+                        Optional.of(authzInfo(randomSubsetOf(randomIntBetween(1, filteredRoles.size()), filteredRoles)
+                            .toArray(new String[0]))),
                         Optional.of(someIndicesDoNotMatch.toArray(new String[0])))));
         assertTrue("Matches the filter predicate because of the empty indices.", auditTrail.eventFilterPolicyRegistry.ignorePredicate()
                 .test(new AuditEventMetaInfo(Optional.of(randomFrom(filteredUsers)), Optional.of(randomFrom(filteredRealms)),
-                        Optional.of(randomSubsetOf(randomIntBetween(1, filteredRoles.size()), filteredRoles).toArray(new String[0])),
+                        Optional.of(authzInfo(
+                            randomSubsetOf(randomIntBetween(1, filteredRoles.size()), filteredRoles).toArray(new String[0]))),
                         Optional.empty())));
         assertTrue("Matches the filter predicate because of the empty indices.", auditTrail.eventFilterPolicyRegistry.ignorePredicate()
                 .test(new AuditEventMetaInfo(Optional.of(randomFrom(filteredUsers)), Optional.of(randomFrom(filteredRealms)),
-                        Optional.of(randomSubsetOf(randomIntBetween(1, filteredRoles.size()), filteredRoles).toArray(new String[0])),
+                        Optional.of(authzInfo(
+                            randomSubsetOf(randomIntBetween(1, filteredRoles.size()), filteredRoles).toArray(new String[0]))),
                         Optional.of(new String[0]))));
         assertTrue("Matches the filter predicate because of the empty indices.", auditTrail.eventFilterPolicyRegistry.ignorePredicate()
                 .test(new AuditEventMetaInfo(Optional.of(randomFrom(filteredUsers)), Optional.of(randomFrom(filteredRealms)),
-                        Optional.of(randomSubsetOf(randomIntBetween(1, filteredRoles.size()), filteredRoles).toArray(new String[0])),
+                        Optional.of(authzInfo(
+                            randomSubsetOf(randomIntBetween(1, filteredRoles.size()), filteredRoles).toArray(new String[0]))),
                         Optional.of(new String[] { null }))));
     }
 
@@ -396,26 +414,28 @@ public class LoggingAuditTrailFilterTests extends ESTestCase {
         assertTrue("Matches both the first and the second filter predicates.",
                 auditTrail.eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.of(randomFrom(filteredUsers)),
                         Optional.of(randomFrom(filteredRealms)),
-                        Optional.of(randomSubsetOf(randomIntBetween(1, filteredRoles.size()), filteredRoles).toArray(new String[0])),
+                        Optional.of(authzInfo(randomSubsetOf(randomIntBetween(1, filteredRoles.size()), filteredRoles)
+                            .toArray(new String[0]))),
                         Optional.of(randomSubsetOf(randomIntBetween(1, filteredIndices.size()), filteredIndices).toArray(new String[0])))));
         // matches first policy but not the second
         assertTrue("Matches the first filter predicate but not the second.",
                 auditTrail.eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.of(unfilteredUser),
                         Optional.of(randomFrom(filteredRealms)),
-                        Optional.of(randomSubsetOf(randomIntBetween(1, filteredRoles.size()), filteredRoles).toArray(new String[0])),
+                        Optional.of(authzInfo(randomSubsetOf(randomIntBetween(1, filteredRoles.size()), filteredRoles)
+                            .toArray(new String[0]))),
                         Optional.of(someIndicesDoNotMatch.toArray(new String[0])))));
         // matches the second policy but not the first
         assertTrue("Matches the second filter predicate but not the first.",
                 auditTrail.eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.of(randomFrom(filteredUsers)),
                         Optional.of(UNFILTER_MARKER + randomAlphaOfLengthBetween(1, 8)),
-                        Optional.of(someRolesDoNotMatch.toArray(new String[0])),
+                        Optional.of(authzInfo(someRolesDoNotMatch.toArray(new String[0]))),
                         Optional.of(randomSubsetOf(randomIntBetween(1, filteredIndices.size()), filteredIndices).toArray(new String[0])))));
         // matches neither the first nor the second policies
         assertFalse("Matches neither the first nor the second filter predicates.",
                 auditTrail.eventFilterPolicyRegistry.ignorePredicate()
                         .test(new AuditEventMetaInfo(Optional.of(unfilteredUser),
                         Optional.of(UNFILTER_MARKER + randomAlphaOfLengthBetween(1, 8)),
-                        Optional.of(someRolesDoNotMatch.toArray(new String[0])),
+                        Optional.of(authzInfo(someRolesDoNotMatch.toArray(new String[0]))),
                         Optional.of(someIndicesDoNotMatch.toArray(new String[0])))));
     }
 
@@ -548,55 +568,61 @@ public class LoggingAuditTrailFilterTests extends ESTestCase {
         threadContext.stashContext();
 
         // accessGranted
-        auditTrail.accessGranted(randomAlphaOfLength(8), unfilteredAuthentication, "_action", message, new String[] { "role1" });
+        auditTrail.accessGranted(randomAlphaOfLength(8), unfilteredAuthentication, "_action", message, authzInfo(new String[] { "role1" }));
         assertThat("AccessGranted message: unfiltered user is filtered out", logOutput.size(), is(1));
         logOutput.clear();
         threadContext.stashContext();
 
-        auditTrail.accessGranted(randomAlphaOfLength(8), filteredAuthentication, "_action", message, new String[] { "role1" });
+        auditTrail.accessGranted(randomAlphaOfLength(8), filteredAuthentication, "_action", message, authzInfo(new String[] { "role1" }));
         assertThat("AccessGranted message: filtered user is not filtered out", logOutput.size(), is(0));
         logOutput.clear();
         threadContext.stashContext();
 
         auditTrail.accessGranted(randomAlphaOfLength(8), createAuthentication(SystemUser.INSTANCE, "effectiveRealmName"),
-            "internal:_action", message, new String[] { "role1" });
+            "internal:_action", message, authzInfo(new String[] { "role1" }));
         assertThat("AccessGranted internal message: system user is filtered out", logOutput.size(), is(1));
         logOutput.clear();
         threadContext.stashContext();
 
-        auditTrail.accessGranted(randomAlphaOfLength(8), unfilteredAuthentication, "internal:_action", message, new String[] { "role1" });
+        auditTrail.accessGranted(randomAlphaOfLength(8), unfilteredAuthentication, "internal:_action", message,
+            authzInfo(new String[] { "role1" }));
         assertThat("AccessGranted internal message: unfiltered user is filtered out", logOutput.size(), is(1));
         logOutput.clear();
         threadContext.stashContext();
 
-        auditTrail.accessGranted(randomAlphaOfLength(8), filteredAuthentication, "internal:_action", message, new String[] { "role1" });
+        auditTrail.accessGranted(randomAlphaOfLength(8), filteredAuthentication, "internal:_action", message,
+            authzInfo(new String[] { "role1" }));
         assertThat("AccessGranted internal message: filtered user is not filtered out", logOutput.size(), is(0));
         logOutput.clear();
         threadContext.stashContext();
 
         // accessDenied
-        auditTrail.accessDenied(randomAlphaOfLength(8), unfilteredAuthentication, "_action", message, new String[] { "role1" });
+        auditTrail.accessDenied(randomAlphaOfLength(8), unfilteredAuthentication, "_action", message,
+            authzInfo(new String[] { "role1" }));
         assertThat("AccessDenied message: unfiltered user is filtered out", logOutput.size(), is(1));
         logOutput.clear();
         threadContext.stashContext();
 
-        auditTrail.accessDenied(randomAlphaOfLength(8), filteredAuthentication, "_action", message, new String[] { "role1" });
+        auditTrail.accessDenied(randomAlphaOfLength(8), filteredAuthentication, "_action", message,
+            authzInfo(new String[] { "role1" }));
         assertThat("AccessDenied message: filtered user is not filtered out", logOutput.size(), is(0));
         logOutput.clear();
         threadContext.stashContext();
 
         auditTrail.accessDenied(randomAlphaOfLength(8), createAuthentication(SystemUser.INSTANCE, "effectiveRealmName"), "internal:_action",
-                message, new String[] { "role1" });
+                message, authzInfo(new String[] { "role1" }));
         assertThat("AccessDenied internal message: system user is filtered out", logOutput.size(), is(1));
         logOutput.clear();
         threadContext.stashContext();
 
-        auditTrail.accessDenied(randomAlphaOfLength(8), unfilteredAuthentication, "internal:_action", message, new String[] { "role1" });
+        auditTrail.accessDenied(randomAlphaOfLength(8), unfilteredAuthentication, "internal:_action", message,
+            authzInfo(new String[] { "role1" }));
         assertThat("AccessDenied internal message: unfiltered user is filtered out", logOutput.size(), is(1));
         logOutput.clear();
         threadContext.stashContext();
 
-        auditTrail.accessDenied(randomAlphaOfLength(8), filteredAuthentication, "internal:_action", message, new String[] { "role1" });
+        auditTrail.accessDenied(randomAlphaOfLength(8), filteredAuthentication, "internal:_action", message,
+            authzInfo(new String[] { "role1" }));
         assertThat("AccessDenied internal message: filtered user is not filtered out", logOutput.size(), is(0));
         logOutput.clear();
         threadContext.stashContext();
@@ -652,36 +678,36 @@ public class LoggingAuditTrailFilterTests extends ESTestCase {
 
         // runAsGranted
         auditTrail.runAsGranted(randomAlphaOfLength(8), unfilteredAuthentication, "_action", new MockMessage(threadContext),
-            new String[] { "role1" });
+            authzInfo(new String[] { "role1" }));
         assertThat("RunAsGranted message: unfiltered user is filtered out", logOutput.size(), is(1));
         logOutput.clear();
         threadContext.stashContext();
 
         auditTrail.runAsGranted(randomAlphaOfLength(8), filteredAuthentication, "_action", new MockMessage(threadContext),
-            new String[] { "role1" });
+            authzInfo(new String[] { "role1" }));
         assertThat("RunAsGranted message: filtered user is not filtered out", logOutput.size(), is(0));
         logOutput.clear();
         threadContext.stashContext();
 
         // runAsDenied
         auditTrail.runAsDenied(randomAlphaOfLength(8), unfilteredAuthentication, "_action", new MockMessage(threadContext),
-            new String[] { "role1" });
+            authzInfo(new String[] { "role1" }));
         assertThat("RunAsDenied message: unfiltered user is filtered out", logOutput.size(), is(1));
         logOutput.clear();
         threadContext.stashContext();
 
         auditTrail.runAsDenied(randomAlphaOfLength(8), filteredAuthentication, "_action", new MockMessage(threadContext),
-            new String[] { "role1" });
+            authzInfo(new String[] { "role1" }));
         assertThat("RunAsDenied message: filtered user is not filtered out", logOutput.size(), is(0));
         logOutput.clear();
         threadContext.stashContext();
 
-        auditTrail.runAsDenied(randomAlphaOfLength(8), unfilteredAuthentication, getRestRequest(), new String[] { "role1" });
+        auditTrail.runAsDenied(randomAlphaOfLength(8), unfilteredAuthentication, getRestRequest(), authzInfo(new String[] { "role1" }));
         assertThat("RunAsDenied rest request: unfiltered user is filtered out", logOutput.size(), is(1));
         logOutput.clear();
         threadContext.stashContext();
 
-        auditTrail.runAsDenied(randomAlphaOfLength(8), filteredAuthentication, getRestRequest(), new String[] { "role1" });
+        auditTrail.runAsDenied(randomAlphaOfLength(8), filteredAuthentication, getRestRequest(), authzInfo(new String[] { "role1" }));
         assertThat("RunAsDenied rest request: filtered user is not filtered out", logOutput.size(), is(0));
         logOutput.clear();
         threadContext.stashContext();
@@ -826,74 +852,74 @@ public class LoggingAuditTrailFilterTests extends ESTestCase {
 
         // accessGranted
         auditTrail.accessGranted(randomAlphaOfLength(8), createAuthentication(user, filteredRealm), "_action", message,
-            new String[] { "role1" });
+            authzInfo(new String[] { "role1" }));
         assertThat("AccessGranted message: filtered realm is not filtered out", logOutput.size(), is(0));
         logOutput.clear();
         threadContext.stashContext();
 
         auditTrail.accessGranted(randomAlphaOfLength(8), createAuthentication(user, unfilteredRealm), "_action", message,
-            new String[] { "role1" });
+            authzInfo(new String[] { "role1" }));
         assertThat("AccessGranted message: unfiltered realm is filtered out", logOutput.size(), is(1));
         logOutput.clear();
         threadContext.stashContext();
 
         auditTrail.accessGranted(randomAlphaOfLength(8), createAuthentication(SystemUser.INSTANCE, filteredRealm), "internal:_action",
-            message, new String[] { "role1" });
+            message, authzInfo(new String[] { "role1" }));
         assertThat("AccessGranted internal message system user: filtered realm is not filtered out", logOutput.size(), is(0));
         logOutput.clear();
         threadContext.stashContext();
 
         auditTrail.accessGranted(randomAlphaOfLength(8), createAuthentication(SystemUser.INSTANCE, unfilteredRealm), "internal:_action",
-            message, new String[] { "role1" });
+            message, authzInfo(new String[] { "role1" }));
         assertThat("AccessGranted internal message system user: unfiltered realm is filtered out", logOutput.size(), is(1));
         logOutput.clear();
         threadContext.stashContext();
 
         auditTrail.accessGranted(randomAlphaOfLength(8), createAuthentication(user, filteredRealm), "internal:_action", message,
-            new String[] { "role1" });
+            authzInfo(new String[] { "role1" }));
         assertThat("AccessGranted internal message: filtered realm is not filtered out", logOutput.size(), is(0));
         logOutput.clear();
         threadContext.stashContext();
 
         auditTrail.accessGranted(randomAlphaOfLength(8), createAuthentication(user, unfilteredRealm), "internal:_action", message,
-            new String[] { "role1" });
+            authzInfo(new String[] { "role1" }));
         assertThat("AccessGranted internal message: unfiltered realm is filtered out", logOutput.size(), is(1));
         logOutput.clear();
         threadContext.stashContext();
 
         // accessDenied
         auditTrail.accessDenied(randomAlphaOfLength(8), createAuthentication(user, filteredRealm), "_action", message,
-            new String[] { "role1" });
+            authzInfo(new String[] { "role1" }));
         assertThat("AccessDenied message: filtered realm is not filtered out", logOutput.size(), is(0));
         logOutput.clear();
         threadContext.stashContext();
 
         auditTrail.accessDenied(randomAlphaOfLength(8), createAuthentication(user, unfilteredRealm), "_action", message,
-            new String[] { "role1" });
+            authzInfo(new String[] { "role1" }));
         assertThat("AccessDenied message: unfiltered realm is filtered out", logOutput.size(), is(1));
         logOutput.clear();
         threadContext.stashContext();
 
         auditTrail.accessDenied(randomAlphaOfLength(8), createAuthentication(SystemUser.INSTANCE, filteredRealm), "internal:_action",
-            message, new String[] { "role1" });
+            message, authzInfo(new String[] { "role1" }));
         assertThat("AccessDenied internal message system user: filtered realm is not filtered out", logOutput.size(), is(0));
         logOutput.clear();
         threadContext.stashContext();
 
         auditTrail.accessDenied(randomAlphaOfLength(8), createAuthentication(SystemUser.INSTANCE, unfilteredRealm), "internal:_action",
-            message, new String[] { "role1" });
+            message, authzInfo(new String[] { "role1" }));
         assertThat("AccessDenied internal message system user: unfiltered realm is filtered out", logOutput.size(), is(1));
         logOutput.clear();
         threadContext.stashContext();
 
         auditTrail.accessDenied(randomAlphaOfLength(8), createAuthentication(user, filteredRealm), "internal:_action", message,
-            new String[] { "role1" });
+            authzInfo(new String[] { "role1" }));
         assertThat("AccessGranted internal message: filtered realm is not filtered out", logOutput.size(), is(0));
         logOutput.clear();
         threadContext.stashContext();
 
         auditTrail.accessDenied(randomAlphaOfLength(8), createAuthentication(user, unfilteredRealm), "internal:_action", message,
-            new String[] { "role1" });
+            authzInfo(new String[] { "role1" }));
         assertThat("AccessGranted internal message: unfiltered realm is filtered out", logOutput.size(), is(1));
         logOutput.clear();
         threadContext.stashContext();
@@ -948,38 +974,38 @@ public class LoggingAuditTrailFilterTests extends ESTestCase {
 
         // runAsGranted
         auditTrail.runAsGranted(randomAlphaOfLength(8), createAuthentication(user, filteredRealm), "_action",
-            new MockMessage(threadContext), new String[] { "role1" });
+            new MockMessage(threadContext), authzInfo(new String[] { "role1" }));
         assertThat("RunAsGranted message: filtered realm is not filtered out", logOutput.size(), is(0));
         logOutput.clear();
         threadContext.stashContext();
 
         auditTrail.runAsGranted(randomAlphaOfLength(8), createAuthentication(user, unfilteredRealm), "_action",
-            new MockMessage(threadContext), new String[] { "role1" });
+            new MockMessage(threadContext), authzInfo(new String[] { "role1" }));
         assertThat("RunAsGranted message: unfiltered realm is filtered out", logOutput.size(), is(1));
         logOutput.clear();
         threadContext.stashContext();
 
         // runAsDenied
         auditTrail.runAsDenied(randomAlphaOfLength(8), createAuthentication(user, filteredRealm), "_action", new MockMessage(threadContext),
-                new String[] { "role1" });
+                authzInfo(new String[] { "role1" }));
         assertThat("RunAsDenied message: filtered realm is not filtered out", logOutput.size(), is(0));
         logOutput.clear();
         threadContext.stashContext();
 
         auditTrail.runAsDenied(randomAlphaOfLength(8), createAuthentication(user, unfilteredRealm), "_action",
-            new MockMessage(threadContext), new String[] { "role1" });
+            new MockMessage(threadContext), authzInfo(new String[] { "role1" }));
         assertThat("RunAsDenied message: unfiltered realm is filtered out", logOutput.size(), is(1));
         logOutput.clear();
         threadContext.stashContext();
 
         auditTrail.runAsDenied(randomAlphaOfLength(8), createAuthentication(user, filteredRealm), getRestRequest(),
-            new String[] { "role1" });
+            authzInfo(new String[] { "role1" }));
         assertThat("RunAsDenied rest request: filtered realm is not filtered out", logOutput.size(), is(0));
         logOutput.clear();
         threadContext.stashContext();
 
         auditTrail.runAsDenied(randomAlphaOfLength(8), createAuthentication(user, unfilteredRealm), getRestRequest(),
-            new String[] { "role1" });
+            authzInfo(new String[] { "role1" }));
         assertThat("RunAsDenied rest request: unfiltered realm is filtered out", logOutput.size(), is(1));
         logOutput.clear();
         threadContext.stashContext();
@@ -1143,67 +1169,67 @@ public class LoggingAuditTrailFilterTests extends ESTestCase {
         threadContext.stashContext();
 
         // accessGranted
-        auditTrail.accessGranted(randomAlphaOfLength(8), authentication, "_action", message, unfilteredRoles);
+        auditTrail.accessGranted(randomAlphaOfLength(8), authentication, "_action", message, authzInfo(unfilteredRoles));
         assertThat("AccessGranted message: unfiltered roles filtered out", logOutput.size(), is(1));
         logOutput.clear();
         threadContext.stashContext();
 
-        auditTrail.accessGranted(randomAlphaOfLength(8), authentication, "_action", message, filteredRoles);
+        auditTrail.accessGranted(randomAlphaOfLength(8), authentication, "_action", message, authzInfo(filteredRoles));
         assertThat("AccessGranted message: filtered roles not filtered out", logOutput.size(), is(0));
         logOutput.clear();
         threadContext.stashContext();
 
         auditTrail.accessGranted(randomAlphaOfLength(8), createAuthentication(SystemUser.INSTANCE, "effectiveRealmName"),
-            "internal:_action", message, unfilteredRoles);
+            "internal:_action", message, authzInfo(unfilteredRoles));
         assertThat("AccessGranted internal message system user: unfiltered roles filtered out", logOutput.size(), is(1));
         logOutput.clear();
         threadContext.stashContext();
 
         auditTrail.accessGranted(randomAlphaOfLength(8), createAuthentication(SystemUser.INSTANCE, "effectiveRealmName"),
-            "internal:_action", message, filteredRoles);
+            "internal:_action", message, authzInfo(filteredRoles));
         assertThat("AccessGranted internal message system user: filtered roles not filtered out", logOutput.size(), is(0));
         logOutput.clear();
         threadContext.stashContext();
 
-        auditTrail.accessGranted(randomAlphaOfLength(8), authentication, "internal:_action", message, unfilteredRoles);
+        auditTrail.accessGranted(randomAlphaOfLength(8), authentication, "internal:_action", message, authzInfo(unfilteredRoles));
         assertThat("AccessGranted internal message: unfiltered roles filtered out", logOutput.size(), is(1));
         logOutput.clear();
         threadContext.stashContext();
 
-        auditTrail.accessGranted(randomAlphaOfLength(8), authentication, "internal:_action", message, filteredRoles);
+        auditTrail.accessGranted(randomAlphaOfLength(8), authentication, "internal:_action", message, authzInfo(filteredRoles));
         assertThat("AccessGranted internal message: filtered roles not filtered out", logOutput.size(), is(0));
         logOutput.clear();
         threadContext.stashContext();
 
         // accessDenied
-        auditTrail.accessDenied(randomAlphaOfLength(8), authentication, "_action", message, unfilteredRoles);
+        auditTrail.accessDenied(randomAlphaOfLength(8), authentication, "_action", message, authzInfo(unfilteredRoles));
         assertThat("AccessDenied message: unfiltered roles filtered out", logOutput.size(), is(1));
         logOutput.clear();
         threadContext.stashContext();
 
-        auditTrail.accessDenied(randomAlphaOfLength(8), authentication, "_action", message, filteredRoles);
+        auditTrail.accessDenied(randomAlphaOfLength(8), authentication, "_action", message, authzInfo(filteredRoles));
         assertThat("AccessDenied message: filtered roles not filtered out", logOutput.size(), is(0));
         logOutput.clear();
         threadContext.stashContext();
 
         auditTrail.accessDenied(randomAlphaOfLength(8), createAuthentication(SystemUser.INSTANCE, "effectiveRealmName"), "internal:_action",
-            message, unfilteredRoles);
+            message, authzInfo(unfilteredRoles));
         assertThat("AccessDenied internal message system user: unfiltered roles filtered out", logOutput.size(), is(1));
         logOutput.clear();
         threadContext.stashContext();
 
         auditTrail.accessDenied(randomAlphaOfLength(8), createAuthentication(SystemUser.INSTANCE, "effectiveRealmName"), "internal:_action",
-            message, filteredRoles);
+            message, authzInfo(filteredRoles));
         assertThat("AccessDenied internal message system user: filtered roles not filtered out", logOutput.size(), is(0));
         logOutput.clear();
         threadContext.stashContext();
 
-        auditTrail.accessDenied(randomAlphaOfLength(8), authentication, "internal:_action", message, unfilteredRoles);
+        auditTrail.accessDenied(randomAlphaOfLength(8), authentication, "internal:_action", message, authzInfo(unfilteredRoles));
         assertThat("AccessDenied internal message: unfiltered roles filtered out", logOutput.size(), is(1));
         logOutput.clear();
         threadContext.stashContext();
 
-        auditTrail.accessDenied(randomAlphaOfLength(8), authentication, "internal:_action", message, filteredRoles);
+        auditTrail.accessDenied(randomAlphaOfLength(8), authentication, "internal:_action", message, authzInfo(filteredRoles));
         assertThat("AccessDenied internal message: filtered roles not filtered out", logOutput.size(), is(0));
         logOutput.clear();
         threadContext.stashContext();
@@ -1229,33 +1255,36 @@ public class LoggingAuditTrailFilterTests extends ESTestCase {
         threadContext.stashContext();
 
         // runAsGranted
-        auditTrail.runAsGranted(randomAlphaOfLength(8), authentication, "_action", new MockMessage(threadContext), unfilteredRoles);
+        auditTrail.runAsGranted(randomAlphaOfLength(8), authentication, "_action", new MockMessage(threadContext),
+            authzInfo(unfilteredRoles));
         assertThat("RunAsGranted message: unfiltered roles filtered out", logOutput.size(), is(1));
         logOutput.clear();
         threadContext.stashContext();
 
-        auditTrail.runAsGranted(randomAlphaOfLength(8), authentication, "_action", new MockMessage(threadContext), filteredRoles);
+        auditTrail.runAsGranted(randomAlphaOfLength(8), authentication, "_action", new MockMessage(threadContext),
+            authzInfo(filteredRoles));
         assertThat("RunAsGranted message: filtered roles not filtered out", logOutput.size(), is(0));
         logOutput.clear();
         threadContext.stashContext();
 
         // runAsDenied
-        auditTrail.runAsDenied(randomAlphaOfLength(8), authentication, "_action", new MockMessage(threadContext), unfilteredRoles);
+        auditTrail.runAsDenied(randomAlphaOfLength(8), authentication, "_action", new MockMessage(threadContext),
+            authzInfo(unfilteredRoles));
         assertThat("RunAsDenied message: unfiltered roles filtered out", logOutput.size(), is(1));
         logOutput.clear();
         threadContext.stashContext();
 
-        auditTrail.runAsDenied(randomAlphaOfLength(8), authentication, "_action", new MockMessage(threadContext), filteredRoles);
+        auditTrail.runAsDenied(randomAlphaOfLength(8), authentication, "_action", new MockMessage(threadContext), authzInfo(filteredRoles));
         assertThat("RunAsDenied message: filtered roles not filtered out", logOutput.size(), is(0));
         logOutput.clear();
         threadContext.stashContext();
 
-        auditTrail.runAsDenied(randomAlphaOfLength(8), authentication, getRestRequest(), unfilteredRoles);
+        auditTrail.runAsDenied(randomAlphaOfLength(8), authentication, getRestRequest(), authzInfo(unfilteredRoles));
         assertThat("RunAsDenied rest request: unfiltered roles filtered out", logOutput.size(), is(1));
         logOutput.clear();
         threadContext.stashContext();
 
-        auditTrail.runAsDenied(randomAlphaOfLength(8), authentication, getRestRequest(), filteredRoles);
+        auditTrail.runAsDenied(randomAlphaOfLength(8), authentication, getRestRequest(), authzInfo(filteredRoles));
         assertThat("RunAsDenied rest request: filtered roles not filtered out", logOutput.size(), is(0));
         logOutput.clear();
         threadContext.stashContext();
@@ -1464,7 +1493,7 @@ public class LoggingAuditTrailFilterTests extends ESTestCase {
         threadContext.stashContext();
 
         // accessGranted
-        auditTrail.accessGranted(randomAlphaOfLength(8), authentication, "_action", noIndexMessage, new String[] { "role1" });
+        auditTrail.accessGranted(randomAlphaOfLength(8), authentication, "_action", noIndexMessage, authzInfo(new String[] { "role1" }));
         if (filterMissingIndices) {
             assertThat("AccessGranted message no index: not filtered out by the missing indices filter", logOutput.size(), is(0));
         } else {
@@ -1475,19 +1504,19 @@ public class LoggingAuditTrailFilterTests extends ESTestCase {
 
         auditTrail.accessGranted(randomAlphaOfLength(8), authentication, "_action",
             new MockIndicesRequest(threadContext, unfilteredIndices),
-                new String[] { "role1" });
+                authzInfo(new String[] { "role1" }));
         assertThat("AccessGranted message unfiltered indices: filtered out by indices filter", logOutput.size(), is(1));
         logOutput.clear();
         threadContext.stashContext();
 
         auditTrail.accessGranted(randomAlphaOfLength(8), authentication, "_action", new MockIndicesRequest(threadContext, filteredIndices),
-                new String[] { "role1" });
+                authzInfo(new String[] { "role1" }));
         assertThat("AccessGranted message filtered indices: not filtered out by indices filter", logOutput.size(), is(0));
         logOutput.clear();
         threadContext.stashContext();
 
         auditTrail.accessGranted(randomAlphaOfLength(8), createAuthentication(SystemUser.INSTANCE, "effectiveRealmName"),
-            "internal:_action", noIndexMessage, new String[] { "role1" });
+            "internal:_action", noIndexMessage, authzInfo(new String[] { "role1" }));
         if (filterMissingIndices) {
             assertThat("AccessGranted message system user no index: not filtered out by the missing indices filter", logOutput.size(),
                     is(0));
@@ -1498,19 +1527,19 @@ public class LoggingAuditTrailFilterTests extends ESTestCase {
         threadContext.stashContext();
 
         auditTrail.accessGranted(randomAlphaOfLength(8), createAuthentication(SystemUser.INSTANCE, "effectiveRealmName"),
-            "internal:_action", new MockIndicesRequest(threadContext, unfilteredIndices), new String[] { "role1" });
+            "internal:_action", new MockIndicesRequest(threadContext, unfilteredIndices), authzInfo(new String[] { "role1" }));
         assertThat("AccessGranted message system user unfiltered indices: filtered out by indices filter", logOutput.size(), is(1));
         logOutput.clear();
         threadContext.stashContext();
 
         auditTrail.accessGranted(randomAlphaOfLength(8), createAuthentication(SystemUser.INSTANCE, "effectiveRealmName"),
-            "internal:_action", new MockIndicesRequest(threadContext, filteredIndices), new String[] { "role1" });
+            "internal:_action", new MockIndicesRequest(threadContext, filteredIndices), authzInfo(new String[] { "role1" }));
         assertThat("AccessGranted message system user filtered indices: not filtered out by indices filter", logOutput.size(), is(0));
         logOutput.clear();
         threadContext.stashContext();
 
         // accessDenied
-        auditTrail.accessDenied(randomAlphaOfLength(8), authentication, "_action", noIndexMessage, new String[] { "role1" });
+        auditTrail.accessDenied(randomAlphaOfLength(8), authentication, "_action", noIndexMessage, authzInfo(new String[] { "role1" }));
         if (filterMissingIndices) {
             assertThat("AccessDenied message no index: not filtered out by the missing indices filter", logOutput.size(), is(0));
         } else {
@@ -1520,19 +1549,19 @@ public class LoggingAuditTrailFilterTests extends ESTestCase {
         threadContext.stashContext();
 
         auditTrail.accessDenied(randomAlphaOfLength(8), authentication, "_action", new MockIndicesRequest(threadContext, unfilteredIndices),
-                new String[] { "role1" });
+                authzInfo(new String[] { "role1" }));
         assertThat("AccessDenied message unfiltered indices: filtered out by indices filter", logOutput.size(), is(1));
         logOutput.clear();
         threadContext.stashContext();
 
         auditTrail.accessDenied(randomAlphaOfLength(8), authentication, "_action", new MockIndicesRequest(threadContext, filteredIndices),
-                new String[] { "role1" });
+                authzInfo(new String[] { "role1" }));
         assertThat("AccessDenied message filtered indices: not filtered out by indices filter", logOutput.size(), is(0));
         logOutput.clear();
         threadContext.stashContext();
 
         auditTrail.accessDenied(randomAlphaOfLength(8), createAuthentication(SystemUser.INSTANCE, "effectiveRealmName"), "internal:_action",
-            noIndexMessage, new String[] { "role1" });
+            noIndexMessage, authzInfo(new String[] { "role1" }));
         if (filterMissingIndices) {
             assertThat("AccessDenied message system user no index: not filtered out by the missing indices filter", logOutput.size(),
                     is(0));
@@ -1544,14 +1573,14 @@ public class LoggingAuditTrailFilterTests extends ESTestCase {
 
         auditTrail.accessDenied(randomAlphaOfLength(8), createAuthentication(SystemUser.INSTANCE, "effectiveRealmName"), "internal:_action",
                 new MockIndicesRequest(threadContext, unfilteredIndices),
-                new String[] { "role1" });
+                authzInfo(new String[] { "role1" }));
         assertThat("AccessDenied message system user unfiltered indices: filtered out by indices filter", logOutput.size(), is(1));
         logOutput.clear();
         threadContext.stashContext();
 
         auditTrail.accessDenied(randomAlphaOfLength(8), createAuthentication(SystemUser.INSTANCE, "effectiveRealmName"), "internal:_action",
                 new MockIndicesRequest(threadContext, filteredIndices),
-                new String[] { "role1" });
+                authzInfo(new String[] { "role1" }));
         assertThat("AccessGranted message system user filtered indices: not filtered out by indices filter", logOutput.size(), is(0));
         logOutput.clear();
         threadContext.stashContext();
@@ -1577,7 +1606,7 @@ public class LoggingAuditTrailFilterTests extends ESTestCase {
         threadContext.stashContext();
 
         // runAsGranted
-        auditTrail.runAsGranted(randomAlphaOfLength(8), authentication, "_action", noIndexMessage, new String[] { "role1" });
+        auditTrail.runAsGranted(randomAlphaOfLength(8), authentication, "_action", noIndexMessage, authzInfo(new String[] { "role1" }));
         if (filterMissingIndices) {
             assertThat("RunAsGranted message no index: not filtered out by missing indices filter", logOutput.size(), is(0));
         } else {
@@ -1587,19 +1616,19 @@ public class LoggingAuditTrailFilterTests extends ESTestCase {
         threadContext.stashContext();
 
         auditTrail.runAsGranted(randomAlphaOfLength(8), authentication, "_action", new MockIndicesRequest(threadContext, unfilteredIndices),
-                new String[] { "role1" });
+                authzInfo(new String[] { "role1" }));
         assertThat("RunAsGranted message unfiltered indices: filtered out by indices filter", logOutput.size(), is(1));
         logOutput.clear();
         threadContext.stashContext();
 
         auditTrail.runAsGranted(randomAlphaOfLength(8), authentication, "_action", new MockIndicesRequest(threadContext, filteredIndices),
-                new String[] { "role1" });
+                authzInfo(new String[] { "role1" }));
         assertThat("RunAsGranted message filtered indices: not filtered out by indices filter", logOutput.size(), is(0));
         logOutput.clear();
         threadContext.stashContext();
 
         // runAsDenied
-        auditTrail.runAsDenied(randomAlphaOfLength(8), authentication, "_action", noIndexMessage, new String[] { "role1" });
+        auditTrail.runAsDenied(randomAlphaOfLength(8), authentication, "_action", noIndexMessage, authzInfo(new String[] { "role1" }));
         if (filterMissingIndices) {
             assertThat("RunAsDenied message no index: not filtered out by missing indices filter", logOutput.size(), is(0));
         } else {
@@ -1609,18 +1638,18 @@ public class LoggingAuditTrailFilterTests extends ESTestCase {
         threadContext.stashContext();
 
         auditTrail.runAsDenied(randomAlphaOfLength(8), authentication, "_action", new MockIndicesRequest(threadContext, unfilteredIndices),
-                new String[] { "role1" });
+                authzInfo(new String[] { "role1" }));
         assertThat("RunAsDenied message unfiltered indices: filtered out by indices filter", logOutput.size(), is(1));
         logOutput.clear();
         threadContext.stashContext();
 
         auditTrail.runAsDenied(randomAlphaOfLength(8), authentication, "_action", new MockIndicesRequest(threadContext, filteredIndices),
-                new String[] { "role1" });
+                authzInfo(new String[] { "role1" }));
         assertThat("RunAsDenied message filtered indices: not filtered out by indices filter", logOutput.size(), is(0));
         logOutput.clear();
         threadContext.stashContext();
 
-        auditTrail.runAsDenied(randomAlphaOfLength(8), authentication, getRestRequest(), new String[] { "role1" });
+        auditTrail.runAsDenied(randomAlphaOfLength(8), authentication, getRestRequest(), authzInfo(new String[] { "role1" }));
         if (filterMissingIndices) {
             assertThat("RunAsDenied rest request: not filtered out by missing indices filter", logOutput.size(), is(0));
         } else {
@@ -1756,5 +1785,7 @@ public class LoggingAuditTrailFilterTests extends ESTestCase {
         }
     }
 
-
+    private static AuthorizationInfo authzInfo(String[] roles) {
+        return () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, roles);
+    }
 }

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java

@@ -40,6 +40,7 @@ import org.elasticsearch.xpack.core.security.user.SystemUser;
 import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.security.audit.AuditTrail;
 import org.elasticsearch.xpack.security.audit.AuditUtil;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationInfo;
 import org.elasticsearch.xpack.security.rest.RemoteHostHeader;
 import org.elasticsearch.xpack.security.transport.filter.IPFilter;
 import org.elasticsearch.xpack.security.transport.filter.SecurityIpFilterRule;
@@ -63,6 +64,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 import java.util.regex.Pattern;
+
+import static org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail.PRINCIPAL_ROLES_FIELD_NAME;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
@@ -484,11 +487,12 @@ public class LoggingAuditTrailTests extends ESTestCase {
 
     public void testAccessGranted() throws Exception {
         final TransportMessage message = randomBoolean() ? new MockMessage(threadContext) : new MockIndicesRequest(threadContext);
-        final String[] roles = randomArray(0, 4, String[]::new, () -> randomBoolean() ? null : randomAlphaOfLengthBetween(1, 4));
+        final String[] expectedRoles = randomArray(0, 4, String[]::new, () -> randomBoolean() ? null : randomAlphaOfLengthBetween(1, 4));
+        final AuthorizationInfo authorizationInfo = () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, expectedRoles);
         final Authentication authentication = createAuthentication();
         final String requestId = randomRequestId();
 
-        auditTrail.accessGranted(requestId, authentication, "_action", message, roles);
+        auditTrail.accessGranted(requestId, authentication, "_action", message, authorizationInfo);
         final MapBuilder<String, String> checkedFields = new MapBuilder<>(commonFields);
         final MapBuilder<String, String[]> checkedArrayFields = new MapBuilder<>();
         checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.TRANSPORT_ORIGIN_FIELD_VALUE)
@@ -496,7 +500,7 @@ public class LoggingAuditTrailTests extends ESTestCase {
                 .put(LoggingAuditTrail.ACTION_FIELD_NAME, "_action")
                 .put(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME, message.getClass().getSimpleName())
                 .put(LoggingAuditTrail.REQUEST_ID_FIELD_NAME, requestId);
-        checkedArrayFields.put(LoggingAuditTrail.PRINCIPAL_ROLES_FIELD_NAME, roles);
+        checkedArrayFields.put(PRINCIPAL_ROLES_FIELD_NAME, (String[]) authorizationInfo.asMap().get(PRINCIPAL_ROLES_FIELD_NAME));
         subject(authentication, checkedFields);
         restOrTransportOrigin(message, threadContext, checkedFields);
         indicesRequest(message, checkedFields, checkedArrayFields);
@@ -511,16 +515,17 @@ public class LoggingAuditTrailTests extends ESTestCase {
                 .put("xpack.security.audit.logfile.events.exclude", "access_granted")
                 .build();
         auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext);
-        auditTrail.accessGranted(requestId, authentication, "_action", message, roles);
+        auditTrail.accessGranted(requestId, authentication, "_action", message, authorizationInfo);
         assertEmptyLog(logger);
     }
 
     public void testAccessGrantedInternalSystemAction() throws Exception {
         final TransportMessage message = randomBoolean() ? new MockMessage(threadContext) : new MockIndicesRequest(threadContext);
-        final String[] roles = randomArray(0, 4, String[]::new, () -> randomBoolean() ? null : randomAlphaOfLengthBetween(1, 4));
+        final String[] expectedRoles = randomArray(0, 4, String[]::new, () -> randomBoolean() ? null : randomAlphaOfLengthBetween(1, 4));
+        final AuthorizationInfo authorizationInfo = () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, expectedRoles);
         final Authentication authentication = new Authentication(SystemUser.INSTANCE, new RealmRef("_reserved", "test", "foo"), null);
         final String requestId = randomRequestId();
-        auditTrail.accessGranted(requestId, authentication, "internal:_action", message, roles);
+        auditTrail.accessGranted(requestId, authentication, "internal:_action", message, authorizationInfo);
         assertEmptyLog(logger);
 
         // test enabled
@@ -529,7 +534,7 @@ public class LoggingAuditTrailTests extends ESTestCase {
                 .put("xpack.security.audit.logfile.events.include", "system_access_granted")
                 .build();
         auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext);
-        auditTrail.accessGranted(requestId, authentication, "internal:_action", message, roles);
+        auditTrail.accessGranted(requestId, authentication, "internal:_action", message, authorizationInfo);
         final MapBuilder<String, String> checkedFields = new MapBuilder<>(commonFields);
         final MapBuilder<String, String[]> checkedArrayFields = new MapBuilder<>();
         checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.TRANSPORT_ORIGIN_FIELD_VALUE)
@@ -539,7 +544,7 @@ public class LoggingAuditTrailTests extends ESTestCase {
                 .put(LoggingAuditTrail.ACTION_FIELD_NAME, "internal:_action")
                 .put(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME, message.getClass().getSimpleName())
                 .put(LoggingAuditTrail.REQUEST_ID_FIELD_NAME, requestId);
-        checkedArrayFields.put(LoggingAuditTrail.PRINCIPAL_ROLES_FIELD_NAME, roles);
+        checkedArrayFields.put(PRINCIPAL_ROLES_FIELD_NAME, (String[]) authorizationInfo.asMap().get(PRINCIPAL_ROLES_FIELD_NAME));
         restOrTransportOrigin(message, threadContext, checkedFields);
         indicesRequest(message, checkedFields, checkedArrayFields);
         opaqueId(threadContext, checkedFields);
@@ -549,11 +554,12 @@ public class LoggingAuditTrailTests extends ESTestCase {
 
     public void testAccessGrantedInternalSystemActionNonSystemUser() throws Exception {
         final TransportMessage message = randomBoolean() ? new MockMessage(threadContext) : new MockIndicesRequest(threadContext);
-        final String[] roles = randomArray(0, 4, String[]::new, () -> randomBoolean() ? null : randomAlphaOfLengthBetween(1, 4));
+        final String[] expectedRoles = randomArray(0, 4, String[]::new, () -> randomBoolean() ? null : randomAlphaOfLengthBetween(1, 4));
+        final AuthorizationInfo authorizationInfo = () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, expectedRoles);
         final Authentication authentication = createAuthentication();
         final String requestId = randomRequestId();
 
-        auditTrail.accessGranted(requestId, authentication, "internal:_action", message, roles);
+        auditTrail.accessGranted(requestId, authentication, "internal:_action", message, authorizationInfo);
         final MapBuilder<String, String> checkedFields = new MapBuilder<>(commonFields);
         final MapBuilder<String, String[]> checkedArrayFields = new MapBuilder<>();
         checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.TRANSPORT_ORIGIN_FIELD_VALUE)
@@ -561,7 +567,7 @@ public class LoggingAuditTrailTests extends ESTestCase {
             .put(LoggingAuditTrail.ACTION_FIELD_NAME, "internal:_action")
             .put(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME, message.getClass().getSimpleName())
             .put(LoggingAuditTrail.REQUEST_ID_FIELD_NAME, requestId);
-        checkedArrayFields.put(LoggingAuditTrail.PRINCIPAL_ROLES_FIELD_NAME, roles);
+        checkedArrayFields.put(PRINCIPAL_ROLES_FIELD_NAME, (String[]) authorizationInfo.asMap().get(PRINCIPAL_ROLES_FIELD_NAME));
         subject(authentication, checkedFields);
         restOrTransportOrigin(message, threadContext, checkedFields);
         indicesRequest(message, checkedFields, checkedArrayFields);
@@ -576,17 +582,18 @@ public class LoggingAuditTrailTests extends ESTestCase {
                     .put("xpack.security.audit.logfile.events.exclude", "access_granted")
                     .build();
         auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext);
-        auditTrail.accessGranted(requestId, authentication, "internal:_action", message, roles);
+        auditTrail.accessGranted(requestId, authentication, "internal:_action", message, authorizationInfo);
         assertEmptyLog(logger);
     }
 
     public void testAccessDenied() throws Exception {
         final TransportMessage message = randomBoolean() ? new MockMessage(threadContext) : new MockIndicesRequest(threadContext);
-        final String[] roles = randomArray(0, 4, String[]::new, () -> randomBoolean() ? null : randomAlphaOfLengthBetween(1, 4));
+        final String[] expectedRoles = randomArray(0, 4, String[]::new, () -> randomBoolean() ? null : randomAlphaOfLengthBetween(1, 4));
+        final AuthorizationInfo authorizationInfo = () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, expectedRoles);
         final Authentication authentication = createAuthentication();
         final String requestId = randomRequestId();
 
-        auditTrail.accessDenied(requestId, authentication, "_action/bar", message, roles);
+        auditTrail.accessDenied(requestId, authentication, "_action/bar", message, authorizationInfo);
         final MapBuilder<String, String> checkedFields = new MapBuilder<>(commonFields);
         final MapBuilder<String, String[]> checkedArrayFields = new MapBuilder<>();
         checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.TRANSPORT_ORIGIN_FIELD_VALUE)
@@ -594,7 +601,7 @@ public class LoggingAuditTrailTests extends ESTestCase {
                 .put(LoggingAuditTrail.ACTION_FIELD_NAME, "_action/bar")
                 .put(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME, message.getClass().getSimpleName())
                 .put(LoggingAuditTrail.REQUEST_ID_FIELD_NAME, requestId);
-        checkedArrayFields.put(LoggingAuditTrail.PRINCIPAL_ROLES_FIELD_NAME, roles);
+        checkedArrayFields.put(PRINCIPAL_ROLES_FIELD_NAME, (String[]) authorizationInfo.asMap().get(PRINCIPAL_ROLES_FIELD_NAME));
         subject(authentication, checkedFields);
         restOrTransportOrigin(message, threadContext, checkedFields);
         indicesRequest(message, checkedFields, checkedArrayFields);
@@ -610,7 +617,7 @@ public class LoggingAuditTrailTests extends ESTestCase {
                 .put("xpack.security.audit.logfile.events.exclude", "access_denied")
                 .build();
         auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext);
-        auditTrail.accessDenied(requestId, authentication, "_action", message, roles);
+        auditTrail.accessDenied(requestId, authentication, "_action", message, authorizationInfo);
         assertEmptyLog(logger);
     }
 
@@ -784,14 +791,15 @@ public class LoggingAuditTrailTests extends ESTestCase {
 
     public void testRunAsGranted() throws Exception {
         final TransportMessage message = randomBoolean() ? new MockMessage(threadContext) : new MockIndicesRequest(threadContext);
-        final String[] roles = randomArray(0, 4, String[]::new, () -> randomBoolean() ? null : randomAlphaOfLengthBetween(1, 4));
+        final String[] expectedRoles = randomArray(0, 4, String[]::new, () -> randomBoolean() ? null : randomAlphaOfLengthBetween(1, 4));
+        final AuthorizationInfo authorizationInfo = () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, expectedRoles);
         final Authentication authentication = new Authentication(
                 new User("running as", new String[] { "r2" }, new User("_username", new String[] { "r1" })),
                 new RealmRef("authRealm", "test", "foo"),
                 new RealmRef("lookRealm", "up", "by"));
         final String requestId = randomRequestId();
 
-        auditTrail.runAsGranted(requestId, authentication, "_action", message, roles);
+        auditTrail.runAsGranted(requestId, authentication, "_action", message, authorizationInfo);
         final MapBuilder<String, String> checkedFields = new MapBuilder<>(commonFields);
         final MapBuilder<String, String[]> checkedArrayFields = new MapBuilder<>();
         checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.TRANSPORT_ORIGIN_FIELD_VALUE)
@@ -803,7 +811,7 @@ public class LoggingAuditTrailTests extends ESTestCase {
                 .put(LoggingAuditTrail.ACTION_FIELD_NAME, "_action")
                 .put(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME, message.getClass().getSimpleName())
                 .put(LoggingAuditTrail.REQUEST_ID_FIELD_NAME, requestId);
-        checkedArrayFields.put(LoggingAuditTrail.PRINCIPAL_ROLES_FIELD_NAME, roles);
+        checkedArrayFields.put(PRINCIPAL_ROLES_FIELD_NAME, (String[]) authorizationInfo.asMap().get(PRINCIPAL_ROLES_FIELD_NAME));
         restOrTransportOrigin(message, threadContext, checkedFields);
         indicesRequest(message, checkedFields, checkedArrayFields);
         opaqueId(threadContext, checkedFields);
@@ -817,20 +825,21 @@ public class LoggingAuditTrailTests extends ESTestCase {
                 .put("xpack.security.audit.logfile.events.exclude", "run_as_granted")
                 .build();
         auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext);
-        auditTrail.runAsGranted(requestId, authentication, "_action", message, roles);
+        auditTrail.runAsGranted(requestId, authentication, "_action", message, authorizationInfo);
         assertEmptyLog(logger);
     }
 
     public void testRunAsDenied() throws Exception {
         final TransportMessage message = randomBoolean() ? new MockMessage(threadContext) : new MockIndicesRequest(threadContext);
-        final String[] roles = randomArray(0, 4, String[]::new, () -> randomAlphaOfLengthBetween(1, 4));
+        final String[] expectedRoles = randomArray(0, 4, String[]::new, () -> randomBoolean() ? null : randomAlphaOfLengthBetween(1, 4));
+        final AuthorizationInfo authorizationInfo = () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, expectedRoles);
         final Authentication authentication = new Authentication(
                 new User("running as", new String[] { "r2" }, new User("_username", new String[] { "r1" })),
                 new RealmRef("authRealm", "test", "foo"),
                 new RealmRef("lookRealm", "up", "by"));
         final String requestId = randomRequestId();
 
-        auditTrail.runAsDenied(requestId, authentication, "_action", message, roles);
+        auditTrail.runAsDenied(requestId, authentication, "_action", message, authorizationInfo);
         final MapBuilder<String, String> checkedFields = new MapBuilder<>(commonFields);
         final MapBuilder<String, String[]> checkedArrayFields = new MapBuilder<>();
         checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.TRANSPORT_ORIGIN_FIELD_VALUE)
@@ -842,7 +851,7 @@ public class LoggingAuditTrailTests extends ESTestCase {
                 .put(LoggingAuditTrail.ACTION_FIELD_NAME, "_action")
                 .put(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME, message.getClass().getSimpleName())
                 .put(LoggingAuditTrail.REQUEST_ID_FIELD_NAME, requestId);
-        checkedArrayFields.put(LoggingAuditTrail.PRINCIPAL_ROLES_FIELD_NAME, roles);
+        checkedArrayFields.put(PRINCIPAL_ROLES_FIELD_NAME, (String[]) authorizationInfo.asMap().get(PRINCIPAL_ROLES_FIELD_NAME));
         restOrTransportOrigin(message, threadContext, checkedFields);
         indicesRequest(message, checkedFields, checkedArrayFields);
         opaqueId(threadContext, checkedFields);
@@ -856,7 +865,7 @@ public class LoggingAuditTrailTests extends ESTestCase {
                 .put("xpack.security.audit.logfile.events.exclude", "run_as_denied")
                 .build();
         auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext);
-        auditTrail.runAsDenied(requestId, authentication, "_action", message, roles);
+        auditTrail.runAsDenied(requestId, authentication, "_action", message, authorizationInfo);
         assertEmptyLog(logger);
     }
 
@@ -962,7 +971,8 @@ public class LoggingAuditTrailTests extends ESTestCase {
                 .build();
         auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext);
         final User user = new User("_username", new String[] { "r1" });
-        final String role = randomAlphaOfLengthBetween(1, 6);
+        final AuthorizationInfo authorizationInfo =
+            () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, new String[] { randomAlphaOfLengthBetween(1, 6) });
         final String realm = randomAlphaOfLengthBetween(1, 6);
         // transport messages without indices
         final TransportMessage[] messages = new TransportMessage[] { new MockMessage(threadContext),
@@ -983,10 +993,10 @@ public class LoggingAuditTrailTests extends ESTestCase {
             auditTrail.authenticationFailed("_req_id", realm, new MockToken(), "_action", message);
             assertThat(output.size(), is(logEntriesCount++));
             assertThat(output.get(logEntriesCount - 2), not(containsString("indices=")));
-            auditTrail.accessGranted("_req_id", createAuthentication(), "_action", message, new String[]{role});
+            auditTrail.accessGranted("_req_id", createAuthentication(), "_action", message, authorizationInfo);
             assertThat(output.size(), is(logEntriesCount++));
             assertThat(output.get(logEntriesCount - 2), not(containsString("indices=")));
-            auditTrail.accessDenied("_req_id", createAuthentication(), "_action", message, new String[]{role});
+            auditTrail.accessDenied("_req_id", createAuthentication(), "_action", message, authorizationInfo);
             assertThat(output.size(), is(logEntriesCount++));
             assertThat(output.get(logEntriesCount - 2), not(containsString("indices=")));
             auditTrail.tamperedRequest("_req_id", "_action", message);
@@ -995,10 +1005,10 @@ public class LoggingAuditTrailTests extends ESTestCase {
             auditTrail.tamperedRequest("_req_id", user, "_action", message);
             assertThat(output.size(), is(logEntriesCount++));
             assertThat(output.get(logEntriesCount - 2), not(containsString("indices=")));
-            auditTrail.runAsGranted("_req_id", createAuthentication(), "_action", message, new String[]{role});
+            auditTrail.runAsGranted("_req_id", createAuthentication(), "_action", message, authorizationInfo);
             assertThat(output.size(), is(logEntriesCount++));
             assertThat(output.get(logEntriesCount - 2), not(containsString("indices=")));
-            auditTrail.runAsDenied("_req_id", createAuthentication(), "_action", message, new String[]{role});
+            auditTrail.runAsDenied("_req_id", createAuthentication(), "_action", message, authorizationInfo);
             assertThat(output.size(), is(logEntriesCount++));
             assertThat(output.get(logEntriesCount - 2), not(containsString("indices=")));
             auditTrail.authenticationSuccess("_req_id", realm, user, "_action", message);
@@ -1047,7 +1057,7 @@ public class LoggingAuditTrailTests extends ESTestCase {
                         .reduce((x, y) -> x + "," + y)
                         .orElse("") + "]";
                 final Pattern logEntryFieldPattern = Pattern.compile(Pattern.quote("\"" + checkArrayField.getKey() + "\":" + quotedValue));
-                assertThat("Field " + checkArrayField.getKey() + " value mismatch. Expected " + quotedValue,
+                assertThat("Field " + checkArrayField.getKey() + " value mismatch. Expected " + quotedValue + ".\nLog line: " + logLine,
                         logEntryFieldPattern.matcher(logLine).find(), is(true));
                 // remove checked field
                 logLine = logEntryFieldPattern.matcher(logLine).replaceFirst("");

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java

@@ -28,13 +28,10 @@ import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
 import org.elasticsearch.xpack.core.security.authc.support.Hasher;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
-import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsCache;
-import org.elasticsearch.xpack.core.security.authz.permission.Role;
-import org.elasticsearch.xpack.core.security.authz.permission.LimitedRole;
 import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege;
 import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
 import org.elasticsearch.xpack.core.security.user.User;
-import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore;
+import org.elasticsearch.xpack.security.authc.ApiKeyService.ApiKeyRoleDescriptors;
 import org.elasticsearch.xpack.security.authz.store.NativePrivilegeStore;
 import org.junit.After;
 import org.junit.Before;
@@ -53,7 +50,6 @@ import static org.hamcrest.Matchers.arrayContaining;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.notNullValue;
 import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
@@ -191,26 +187,14 @@ public class ApiKeyServiceTests extends ESTestCase {
 
         final Authentication authentication = new Authentication(new User("joe"), new RealmRef("apikey", "apikey", "node"), null,
             Version.CURRENT, AuthenticationType.API_KEY, authMetadata);
-        CompositeRolesStore rolesStore = mock(CompositeRolesStore.class);
-        doAnswer(invocationOnMock -> {
-            ActionListener<Role> listener = (ActionListener<Role>) invocationOnMock.getArguments()[2];
-            Collection<RoleDescriptor> descriptors = (Collection<RoleDescriptor>) invocationOnMock.getArguments()[0];
-            if (descriptors.size() != 1) {
-                listener.onFailure(new IllegalStateException("descriptors was empty!"));
-            } else if (descriptors.iterator().next().getName().equals("superuser")) {
-                listener.onResponse(ReservedRolesStore.SUPERUSER_ROLE);
-            } else {
-                listener.onFailure(new IllegalStateException("unexpected role name " + descriptors.iterator().next().getName()));
-            }
-            return Void.TYPE;
-        }).when(rolesStore).buildAndCacheRoleFromDescriptors(any(Collection.class), any(String.class), any(ActionListener.class));
         ApiKeyService service = new ApiKeyService(Settings.EMPTY, Clock.systemUTC(), null, null,
-                ClusterServiceUtils.createClusterService(threadPool), rolesStore);
+                ClusterServiceUtils.createClusterService(threadPool));
 
-        PlainActionFuture<Role> roleFuture = new PlainActionFuture<>();
-        service.getRoleForApiKey(authentication, rolesStore, roleFuture);
-        Role role = roleFuture.get();
-        assertThat(role.names(), arrayContaining("superuser"));
+        PlainActionFuture<ApiKeyRoleDescriptors> roleFuture = new PlainActionFuture<>();
+        service.getRoleForApiKey(authentication, roleFuture);
+        ApiKeyRoleDescriptors result = roleFuture.get();
+        assertThat(result.getRoleDescriptors().size(), is(1));
+        assertThat(result.getRoleDescriptors().get(0).getName(), is("superuser"));
     }
 
     public void testGetRolesForApiKey() throws Exception {
@@ -257,39 +241,21 @@ public class ApiKeyServiceTests extends ESTestCase {
                 return null;
             }
         ).when(privilegesStore).getPrivileges(any(Collection.class), any(Collection.class), any(ActionListener.class));
-
-        CompositeRolesStore rolesStore = mock(CompositeRolesStore.class);
-        doAnswer(invocationOnMock -> {
-            ActionListener<Role> listener = (ActionListener<Role>) invocationOnMock.getArguments()[2];
-            Collection<RoleDescriptor> descriptors = (Collection<RoleDescriptor>) invocationOnMock.getArguments()[0];
-            if (descriptors.size() != 1) {
-                listener.onFailure(new IllegalStateException("descriptors was empty!"));
-            } else if (descriptors.iterator().next().getName().equals("a role")) {
-                CompositeRolesStore.buildRoleFromDescriptors(descriptors, new FieldPermissionsCache(Settings.EMPTY),
-                        privilegesStore, ActionListener.wrap(r -> listener.onResponse(r), listener::onFailure));
-            } else if (descriptors.iterator().next().getName().equals("limited role")) {
-                CompositeRolesStore.buildRoleFromDescriptors(descriptors, new FieldPermissionsCache(Settings.EMPTY),
-                        privilegesStore, ActionListener.wrap(r -> listener.onResponse(r), listener::onFailure));
-            } else {
-                listener.onFailure(new IllegalStateException("unexpected role name " + descriptors.iterator().next().getName()));
-            }
-            return Void.TYPE;
-        }).when(rolesStore).buildAndCacheRoleFromDescriptors(any(Collection.class), any(String.class), any(ActionListener.class));
         ApiKeyService service = new ApiKeyService(Settings.EMPTY, Clock.systemUTC(), null, null,
-                ClusterServiceUtils.createClusterService(threadPool), rolesStore);
+                ClusterServiceUtils.createClusterService(threadPool));
 
-        PlainActionFuture<Role> roleFuture = new PlainActionFuture<>();
-        service.getRoleForApiKey(authentication, rolesStore, roleFuture);
-        Role role = roleFuture.get();
+        PlainActionFuture<ApiKeyRoleDescriptors> roleFuture = new PlainActionFuture<>();
+        service.getRoleForApiKey(authentication, roleFuture);
+        ApiKeyRoleDescriptors result = roleFuture.get();
         if (emptyApiKeyRoleDescriptor) {
-            assertThat(role, instanceOf(Role.class));
-            assertThat(role.names(), arrayContaining("limited role"));
+            assertNull(result.getLimitedByRoleDescriptors());
+            assertThat(result.getRoleDescriptors().size(), is(1));
+            assertThat(result.getRoleDescriptors().get(0).getName(), is("limited role"));
         } else {
-            assertThat(role, instanceOf(LimitedRole.class));
-            LimitedRole limitedRole = (LimitedRole) role;
-            assertThat(limitedRole.names(), arrayContaining("a role"));
-            assertThat(limitedRole.limitedBy(), is(notNullValue()));
-            assertThat(limitedRole.limitedBy().names(), arrayContaining("limited role"));
+            assertThat(result.getRoleDescriptors().size(), is(1));
+            assertThat(result.getLimitedByRoleDescriptors().size(), is(1));
+            assertThat(result.getRoleDescriptors().get(0).getName(), is("a role"));
+            assertThat(result.getLimitedByRoleDescriptors().get(0).getName(), is("limited role"));
         }
     }
 }

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java

@@ -66,7 +66,7 @@ import org.elasticsearch.xpack.core.security.authc.Realm;
 import org.elasticsearch.xpack.core.security.authc.Realm.Factory;
 import org.elasticsearch.xpack.core.security.authc.support.Hasher;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
-import org.elasticsearch.xpack.core.security.authz.permission.Role;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.EmptyAuthorizationInfo;
 import org.elasticsearch.xpack.core.security.user.AnonymousUser;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
 import org.elasticsearch.xpack.core.security.user.User;
@@ -74,7 +74,6 @@ import org.elasticsearch.xpack.security.audit.AuditTrailService;
 import org.elasticsearch.xpack.security.audit.AuditUtil;
 import org.elasticsearch.xpack.security.authc.AuthenticationService.Authenticator;
 import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm;
-import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 import org.junit.After;
 import org.junit.Before;
@@ -211,8 +210,7 @@ public class AuthenticationServiceTests extends ESTestCase {
             return null;
         }).when(securityIndex).checkIndexVersionThenExecute(any(Consumer.class), any(Runnable.class));
         ClusterService clusterService = ClusterServiceUtils.createClusterService(threadPool);
-        apiKeyService = new ApiKeyService(settings, Clock.systemUTC(), client, securityIndex, clusterService,
-                mock(CompositeRolesStore.class));
+        apiKeyService = new ApiKeyService(settings, Clock.systemUTC(), client, securityIndex, clusterService);
         tokenService = new TokenService(settings, Clock.systemUTC(), client, securityIndex, clusterService);
         service = new AuthenticationService(settings, realms, auditTrail, new DefaultAuthenticationFailureHandler(Collections.emptyMap()),
             threadPool, new AnonymousUser(settings), tokenService, apiKeyService);
@@ -1022,7 +1020,7 @@ public class AuthenticationServiceTests extends ESTestCase {
             fail("exception should be thrown");
         } catch (ElasticsearchException e) {
             String reqId = expectAuditRequestId();
-            verify(auditTrail).runAsDenied(eq(reqId), any(Authentication.class), eq(restRequest), eq(Role.EMPTY.names()));
+            verify(auditTrail).runAsDenied(eq(reqId), any(Authentication.class), eq(restRequest), eq(EmptyAuthorizationInfo.INSTANCE));
             verifyNoMoreInteractions(auditTrail);
         }
     }
@@ -1041,7 +1039,8 @@ public class AuthenticationServiceTests extends ESTestCase {
             authenticateBlocking("_action", message, null);
             fail("exception should be thrown");
         } catch (ElasticsearchException e) {
-            verify(auditTrail).runAsDenied(eq(reqId), any(Authentication.class), eq("_action"), eq(message), eq(Role.EMPTY.names()));
+            verify(auditTrail).runAsDenied(eq(reqId), any(Authentication.class), eq("_action"), eq(message),
+                eq(EmptyAuthorizationInfo.INSTANCE));
             verifyNoMoreInteractions(auditTrail);
         }
     }

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizedIndicesTests.java

@@ -36,9 +36,9 @@ import static org.hamcrest.Matchers.contains;
 public class AuthorizedIndicesTests extends ESTestCase {
 
     public void testAuthorizedIndicesUserWithoutRoles() {
-        AuthorizedIndices authorizedIndices = new AuthorizedIndices(Role.EMPTY, "", MetaData.EMPTY_META_DATA);
-        List<String> list = authorizedIndices.get();
-        assertTrue(list.isEmpty());
+        List<String> authorizedIndices =
+            RBACEngine.resolveAuthorizedIndicesFromRole(Role.EMPTY, "", MetaData.EMPTY_META_DATA.getAliasAndIndexLookup());
+        assertTrue(authorizedIndices.isEmpty());
     }
 
     public void testAuthorizedIndicesUserWithSomeRoles() {
@@ -70,8 +70,8 @@ public class AuthorizedIndicesTests extends ESTestCase {
         final Set<RoleDescriptor> descriptors = Sets.newHashSet(aStarRole, bRole);
         CompositeRolesStore.buildRoleFromDescriptors(descriptors, new FieldPermissionsCache(Settings.EMPTY), null, future);
         Role roles = future.actionGet();
-        AuthorizedIndices authorizedIndices = new AuthorizedIndices(roles, SearchAction.NAME, metaData);
-        List<String> list = authorizedIndices.get();
+        List<String> list =
+            RBACEngine.resolveAuthorizedIndicesFromRole(roles, SearchAction.NAME, metaData.getAliasAndIndexLookup());
         assertThat(list, containsInAnyOrder("a1", "a2", "aaaaaa", "b", "ab"));
         assertFalse(list.contains("bbbbb"));
         assertFalse(list.contains("ba"));
@@ -81,9 +81,16 @@ public class AuthorizedIndicesTests extends ESTestCase {
 
     public void testAuthorizedIndicesUserWithSomeRolesEmptyMetaData() {
         Role role = Role.builder("role").add(IndexPrivilege.ALL, "*").build();
-        AuthorizedIndices authorizedIndices = new AuthorizedIndices(role, SearchAction.NAME, MetaData.EMPTY_META_DATA);
-        List<String> list = authorizedIndices.get();
-        assertTrue(list.isEmpty());
+        List<String> authorizedIndices =
+            RBACEngine.resolveAuthorizedIndicesFromRole(role, SearchAction.NAME, MetaData.EMPTY_META_DATA.getAliasAndIndexLookup());
+        assertTrue(authorizedIndices.isEmpty());
+    }
+
+    public void testSecurityIndicesAreRemovedFromRegularUser() {
+        Role role = Role.builder("user_role").add(IndexPrivilege.ALL, "*").cluster(ClusterPrivilege.ALL).build();
+        List<String> authorizedIndices =
+            RBACEngine.resolveAuthorizedIndicesFromRole(role, SearchAction.NAME, MetaData.EMPTY_META_DATA.getAliasAndIndexLookup());
+        assertTrue(authorizedIndices.isEmpty());
     }
 
     public void testSecurityIndicesAreRestrictedForDefaultRole() {
@@ -103,11 +110,11 @@ public class AuthorizedIndicesTests extends ESTestCase {
                         .build(), true)
                 .build();
 
-        AuthorizedIndices authorizedIndices = new AuthorizedIndices(role, SearchAction.NAME, metaData);
-        List<String> list = authorizedIndices.get();
-        assertThat(list, containsInAnyOrder("an-index", "another-index"));
-        assertThat(list, not(contains(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX)));
-        assertThat(list, not(contains(RestrictedIndicesNames.SECURITY_INDEX_NAME)));
+        List<String> authorizedIndices =
+            RBACEngine.resolveAuthorizedIndicesFromRole(role, SearchAction.NAME, metaData.getAliasAndIndexLookup());
+        assertThat(authorizedIndices, containsInAnyOrder("an-index", "another-index"));
+        assertThat(authorizedIndices, not(contains(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX)));
+        assertThat(authorizedIndices, not(contains(RestrictedIndicesNames.SECURITY_INDEX_NAME)));
     }
 
     public void testSecurityIndicesAreNotRemovedFromUnrestrictedRole() {
@@ -127,14 +134,14 @@ public class AuthorizedIndicesTests extends ESTestCase {
                         .build(), true)
                 .build();
 
-        AuthorizedIndices authorizedIndices = new AuthorizedIndices(role, SearchAction.NAME, metaData);
-        List<String> list = authorizedIndices.get();
-        assertThat(list, containsInAnyOrder("an-index", "another-index", SecurityIndexManager.SECURITY_INDEX_NAME,
-                SecurityIndexManager.INTERNAL_SECURITY_INDEX));
+        List<String> authorizedIndices =
+            RBACEngine.resolveAuthorizedIndicesFromRole(role, SearchAction.NAME, metaData.getAliasAndIndexLookup());
+        assertThat(authorizedIndices, containsInAnyOrder(
+            "an-index", "another-index", SecurityIndexManager.SECURITY_INDEX_NAME, SecurityIndexManager.INTERNAL_SECURITY_INDEX));
 
-        AuthorizedIndices authorizedIndicesSuperUser = new AuthorizedIndices(ReservedRolesStore.SUPERUSER_ROLE, SearchAction.NAME,
-                metaData);
-        assertThat(authorizedIndicesSuperUser.get(), containsInAnyOrder("an-index", "another-index",
-                SecurityIndexManager.SECURITY_INDEX_NAME, SecurityIndexManager.INTERNAL_SECURITY_INDEX));
+        List<String> authorizedIndicesSuperUser =
+            RBACEngine.resolveAuthorizedIndicesFromRole(role, SearchAction.NAME, metaData.getAliasAndIndexLookup());
+        assertThat(authorizedIndicesSuperUser, containsInAnyOrder(
+            "an-index", "another-index", SecurityIndexManager.SECURITY_INDEX_NAME, SecurityIndexManager.INTERNAL_SECURITY_INDEX));
     }
 }

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java

@@ -49,25 +49,20 @@ import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.protocol.xpack.graph.GraphExploreRequest;
 import org.elasticsearch.search.internal.ShardSearchTransportRequest;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.xpack.core.graph.action.GraphExploreAction;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef;
-import org.elasticsearch.xpack.core.security.authc.DefaultAuthenticationFailureHandler;
 import org.elasticsearch.xpack.core.security.authz.IndicesAndAliasesResolverField;
+import org.elasticsearch.xpack.core.security.authz.ResolvedIndices;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.IndicesPrivileges;
 import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsCache;
 import org.elasticsearch.xpack.core.security.authz.permission.Role;
 import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
-import org.elasticsearch.xpack.core.security.user.AnonymousUser;
 import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.core.security.user.XPackSecurityUser;
 import org.elasticsearch.xpack.core.security.user.XPackUser;
-import org.elasticsearch.xpack.security.audit.AuditTrailService;
-import org.elasticsearch.xpack.security.authc.ApiKeyService;
-import org.elasticsearch.xpack.security.authz.IndicesAndAliasesResolver.ResolvedIndices;
 import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 import org.elasticsearch.xpack.security.test.SecurityTestUtils;
@@ -77,7 +72,6 @@ import org.joda.time.format.DateTimeFormat;
 import org.junit.Before;
 
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -96,6 +90,7 @@ import static org.hamcrest.Matchers.hasItems;
 import static org.hamcrest.Matchers.not;
 import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doCallRealMethod;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -106,10 +101,10 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
     private User userNoIndices;
     private CompositeRolesStore rolesStore;
     private MetaData metaData;
-    private AuthorizationService authzService;
     private IndicesAndAliasesResolver defaultIndicesResolver;
     private IndexNameExpressionResolver indexNameExpressionResolver;
     private Map<String, RoleDescriptor> roleMap;
+    private FieldPermissionsCache fieldPermissionsCache;
 
     @Before
     public void setup() {
@@ -149,6 +144,7 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
             metaData = SecurityTestUtils.addAliasToMetaData(metaData, securityIndexName);
         }
         this.metaData = metaData;
+        this.fieldPermissionsCache = new FieldPermissionsCache(settings);
 
         user = new User("user", "role");
         userDashIndices = new User("dash", "dash");
@@ -168,33 +164,31 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
         roleMap.put(ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR.getName(), ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR);
         final FieldPermissionsCache fieldPermissionsCache = new FieldPermissionsCache(Settings.EMPTY);
         doAnswer((i) -> {
-                ActionListener callback =
-                        (ActionListener) i.getArguments()[1];
-                Set<String> names = (Set<String>) i.getArguments()[0];
-                assertNotNull(names);
-                Set<RoleDescriptor> roleDescriptors = new HashSet<>();
-                for (String name : names) {
-                    RoleDescriptor descriptor = roleMap.get(name);
-                    if (descriptor != null) {
-                        roleDescriptors.add(descriptor);
-                    }
+            ActionListener callback =
+                    (ActionListener) i.getArguments()[1];
+            Set<String> names = (Set<String>) i.getArguments()[0];
+            assertNotNull(names);
+            Set<RoleDescriptor> roleDescriptors = new HashSet<>();
+            for (String name : names) {
+                RoleDescriptor descriptor = roleMap.get(name);
+                if (descriptor != null) {
+                    roleDescriptors.add(descriptor);
                 }
+            }
 
-                if (roleDescriptors.isEmpty()) {
-                    callback.onResponse(Role.EMPTY);
-                } else {
-                    CompositeRolesStore.buildRoleFromDescriptors(roleDescriptors, fieldPermissionsCache, null,
-                            ActionListener.wrap(r -> callback.onResponse(r), callback::onFailure)
-                    );
-                }
-                return Void.TYPE;
-            }).when(rolesStore).roles(any(Set.class), any(ActionListener.class));
+            if (roleDescriptors.isEmpty()) {
+                callback.onResponse(Role.EMPTY);
+            } else {
+                CompositeRolesStore.buildRoleFromDescriptors(roleDescriptors, fieldPermissionsCache, null,
+                        ActionListener.wrap(r -> callback.onResponse(r), callback::onFailure)
+                );
+            }
+            return Void.TYPE;
+        }).when(rolesStore).roles(any(Set.class), any(ActionListener.class));
+        doCallRealMethod().when(rolesStore).getRoles(any(User.class), any(Authentication.class), any(ActionListener.class));
 
         ClusterService clusterService = mock(ClusterService.class);
         when(clusterService.getClusterSettings()).thenReturn(new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
-        authzService = new AuthorizationService(settings, rolesStore, clusterService,
-                mock(AuditTrailService.class), new DefaultAuthenticationFailureHandler(Collections.emptyMap()), mock(ThreadPool.class),
-                new AnonymousUser(settings), mock(ApiKeyService.class), new FieldPermissionsCache(settings));
         defaultIndicesResolver = new IndicesAndAliasesResolver(settings, clusterService);
     }
 
@@ -583,7 +577,7 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
     public void testSearchWithRemoteAndLocalWildcards() {
         SearchRequest request = new SearchRequest("*:foo", "r*:bar*", "remote:baz*", "bar*", "foofoo");
         request.indicesOptions(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), true, false));
-        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME);
+        final List<String> authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME);
         final ResolvedIndices resolved = resolveIndices(request, authorizedIndices);
         assertThat(resolved.getRemote(), containsInAnyOrder("remote:foo", "other_remote:foo", "remote:bar*", "remote:baz*"));
         assertThat(resolved.getLocal(), containsInAnyOrder("bar", "foofoo"));
@@ -701,7 +695,7 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
         IndicesAliasesRequest request = new IndicesAliasesRequest();
         request.addAliasAction(AliasActions.remove().index("foo").alias("foofoobar"));
         request.addAliasAction(AliasActions.remove().index("foofoo").alias("barbaz"));
-        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, IndicesAliasesAction.NAME);
+        final List<String> authorizedIndices = buildAuthorizedIndices(user, IndicesAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         //the union of all indices and aliases gets returned
         String[] expectedIndices = new String[]{"foo", "foofoobar", "foofoo", "barbaz"};
@@ -717,7 +711,7 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
         IndicesAliasesRequest request = new IndicesAliasesRequest();
         request.addAliasAction(AliasActions.remove().index("foo").alias("foofoobar"));
         request.addAliasAction(AliasActions.remove().index("missing_index").alias("missing_alias"));
-        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, IndicesAliasesAction.NAME);
+        final List<String> authorizedIndices = buildAuthorizedIndices(user, IndicesAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         //the union of all indices and aliases gets returned, doesn't matter is some of them don't exist
         String[] expectedIndices = new String[]{"foo", "foofoobar", "missing_index", "missing_alias"};
@@ -733,7 +727,7 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
         IndicesAliasesRequest request = new IndicesAliasesRequest();
         request.addAliasAction(AliasActions.remove().index("foo*").alias("foofoobar"));
         request.addAliasAction(AliasActions.remove().index("bar*").alias("barbaz"));
-        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, IndicesAliasesAction.NAME);
+        final List<String> authorizedIndices = buildAuthorizedIndices(user, IndicesAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         //union of all resolved indices and aliases gets returned, based on what user is authorized for
         String[] expectedIndices = new String[]{"foofoobar", "foofoo", "bar", "barbaz"};
@@ -750,7 +744,7 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
         IndicesAliasesRequest request = new IndicesAliasesRequest();
         request.addAliasAction(AliasActions.remove().index("*").alias("foo*"));
         request.addAliasAction(AliasActions.remove().index("*bar").alias("foo*"));
-        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, IndicesAliasesAction.NAME);
+        final List<String> authorizedIndices = buildAuthorizedIndices(user, IndicesAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         //union of all resolved indices and aliases gets returned, based on what user is authorized for
         //note that the index side will end up containing matching aliases too, which is fine, as es core would do
@@ -768,7 +762,7 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
         IndicesAliasesRequest request = new IndicesAliasesRequest();
         request.addAliasAction(AliasActions.remove().index("*").alias("_all"));
         request.addAliasAction(AliasActions.remove().index("_all").aliases("_all", "explicit"));
-        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, IndicesAliasesAction.NAME);
+        final List<String> authorizedIndices = buildAuthorizedIndices(user, IndicesAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         //union of all resolved indices and aliases gets returned, based on what user is authorized for
         //note that the index side will end up containing matching aliases too, which is fine, as es core would do
@@ -796,7 +790,7 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
         IndicesAliasesRequest request = new IndicesAliasesRequest();
         request.addAliasAction(AliasActions.remove().index("foo*").alias("foofoobar"));
         request.addAliasAction(AliasActions.add().index("bar*").alias("foofoobar"));
-        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, IndicesAliasesAction.NAME);
+        final List<String> authorizedIndices = buildAuthorizedIndices(user, IndicesAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         //union of all resolved indices and aliases gets returned, based on what user is authorized for
         String[] expectedIndices = new String[]{"foofoobar", "foofoo", "bar"};
@@ -811,7 +805,7 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
     public void testResolveGetAliasesRequestStrict() {
         GetAliasesRequest request = new GetAliasesRequest("alias1").indices("foo", "foofoo");
         request.indicesOptions(IndicesOptions.fromOptions(false, randomBoolean(), randomBoolean(), randomBoolean()));
-        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
+        final List<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         //the union of all indices and aliases gets returned
         String[] expectedIndices = new String[]{"alias1", "foo", "foofoo"};
@@ -824,7 +818,7 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
     public void testResolveGetAliasesRequestIgnoreUnavailable() {
         GetAliasesRequest request = new GetAliasesRequest("alias1").indices("foo", "foofoo");
         request.indicesOptions(IndicesOptions.fromOptions(true, randomBoolean(), randomBoolean(), randomBoolean()));
-        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
+        final List<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         String[] expectedIndices = new String[]{"alias1", "foofoo"};
         assertThat(indices.size(), equalTo(expectedIndices.length));
@@ -838,7 +832,7 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
         request.indicesOptions(IndicesOptions.fromOptions(false, randomBoolean(), true, randomBoolean()));
         request.indices("missing");
         request.aliases("alias2");
-        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
+        final List<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         //the union of all indices and aliases gets returned, missing is not an existing index/alias but that doesn't make any difference
         String[] expectedIndices = new String[]{"alias2", "missing"};
@@ -871,7 +865,7 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
         request.indicesOptions(IndicesOptions.fromOptions(false, randomBoolean(), randomBoolean(), randomBoolean()));
         request.indices("missing");
         request.aliases("alias2");
-        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
+        final List<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         String[] expectedIndices = new String[]{"alias2", "missing"};
         assertThat(indices.size(), equalTo(expectedIndices.length));
@@ -885,7 +879,7 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
         request.indicesOptions(IndicesOptions.fromOptions(false, randomBoolean(), true, true));
         request.aliases("alias1");
         request.indices("foo*");
-        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
+        final List<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         //the union of all resolved indices and aliases gets returned, based on indices and aliases that user is authorized for
         String[] expectedIndices = new String[]{"alias1", "foofoo", "foofoo-closed", "foofoobar", "foobarfoo"};
@@ -901,7 +895,7 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
         request.indicesOptions(IndicesOptions.fromOptions(false, randomBoolean(), true, false));
         request.aliases("alias1");
         request.indices("foo*");
-        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
+        final List<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         //the union of all resolved indices and aliases gets returned, based on indices and aliases that user is authorized for
         String[] expectedIndices = new String[]{"alias1", "foofoo", "foofoobar", "foobarfoo"};
@@ -917,7 +911,7 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
         request.indicesOptions(IndicesOptions.fromOptions(true, randomBoolean(), true, false));
         request.aliases("alias1");
         request.indices("foo*", "bar", "missing");
-        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
+        final List<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         //the union of all resolved indices and aliases gets returned, based on indices and aliases that user is authorized for
         String[] expectedIndices = new String[]{"alias1", "foofoo", "foofoobar", "foobarfoo", "bar"};
@@ -953,7 +947,7 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
             request.indices("_all");
         }
         request.aliases("alias1");
-        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
+        final List<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         //the union of all resolved indices and aliases gets returned
         String[] expectedIndices = new String[]{"bar", "bar-closed", "foofoobar", "foobarfoo", "foofoo", "foofoo-closed", "alias1"};
@@ -974,7 +968,7 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
             request.indices("_all");
         }
         request.aliases("alias1");
-        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
+        final List<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         //the union of all resolved indices and aliases gets returned
         String[] expectedIndices = new String[]{"bar", "foofoobar", "foobarfoo", "foofoo", "alias1"};
@@ -1033,7 +1027,7 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
         if (randomBoolean()) {
             request.indices("_all");
         }
-        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
+        final List<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         //the union of all resolved indices and aliases gets returned
         String[] expectedIndices = new String[]{"bar", "bar-closed", "foofoobar", "foobarfoo", "foofoo", "foofoo-closed"};
@@ -1048,7 +1042,7 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
         if (randomBoolean()) {
             request.indices("_all");
         }
-        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
+        final List<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         //the union of all resolved indices and aliases gets returned
         String[] expectedIndices = new String[]{"bar", "bar-closed", "foofoobar", "foobarfoo", "foofoo", "foofoo-closed", "explicit"};
@@ -1063,7 +1057,7 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
         if (randomBoolean()) {
             request.indices("_all");
         }
-        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
+        final List<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         //the union of all resolved indices and aliases gets returned
         String[] expectedIndices = new String[]{"bar", "bar-closed", "foofoobar", "foobarfoo", "foofoo", "foofoo-closed"};
@@ -1077,7 +1071,7 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
         GetAliasesRequest request = new GetAliasesRequest();
         request.indices("*bar");
         request.aliases("foo*");
-        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
+        final List<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         //union of all resolved indices and aliases gets returned, based on what user is authorized for
         //note that the index side will end up containing matching aliases too, which is fine, as es core would do
@@ -1101,7 +1095,7 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
     public void testResolveAliasesExclusionWildcardsGetAliasesRequest() {
         GetAliasesRequest request = new GetAliasesRequest();
         request.aliases("foo*","-foobar*");
-        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
+        final List<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         //union of all resolved indices and aliases gets returned, based on what user is authorized for
         //note that the index side will end up containing matching aliases too, which is fine, as es core would do
@@ -1180,7 +1174,7 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
     }
 
     public void testResolveAdminAction() {
-        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, DeleteIndexAction.NAME);
+        final List<String> authorizedIndices = buildAuthorizedIndices(user, DeleteIndexAction.NAME);
         {
             RefreshRequest request = new RefreshRequest("*");
             List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
@@ -1224,14 +1218,14 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
     public void testXPackSecurityUserHasAccessToSecurityIndex() {
         SearchRequest request = new SearchRequest();
         {
-            final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(XPackSecurityUser.INSTANCE, SearchAction.NAME);
+            final List<String> authorizedIndices = buildAuthorizedIndices(XPackSecurityUser.INSTANCE, SearchAction.NAME);
             List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
             assertThat(indices, hasItem(SecurityIndexManager.SECURITY_INDEX_NAME));
         }
         {
             IndicesAliasesRequest aliasesRequest = new IndicesAliasesRequest();
             aliasesRequest.addAliasAction(AliasActions.add().alias("security_alias").index(SECURITY_INDEX_NAME));
-            final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(XPackSecurityUser.INSTANCE, IndicesAliasesAction.NAME);
+            final List<String> authorizedIndices = buildAuthorizedIndices(XPackSecurityUser.INSTANCE, IndicesAliasesAction.NAME);
             List<String> indices = resolveIndices(aliasesRequest, authorizedIndices).getLocal();
             assertThat(indices, hasItem(SecurityIndexManager.SECURITY_INDEX_NAME));
         }
@@ -1239,7 +1233,7 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
 
     public void testXPackUserDoesNotHaveAccessToSecurityIndex() {
         SearchRequest request = new SearchRequest();
-        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(XPackUser.INSTANCE, SearchAction.NAME);
+        final List<String> authorizedIndices = buildAuthorizedIndices(XPackUser.INSTANCE, SearchAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         assertThat(indices, not(hasItem(SecurityIndexManager.SECURITY_INDEX_NAME)));
     }
@@ -1251,7 +1245,7 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
 
         {
             SearchRequest request = new SearchRequest();
-            final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(allAccessUser, SearchAction.NAME);
+            final List<String> authorizedIndices = buildAuthorizedIndices(allAccessUser, SearchAction.NAME);
             List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
             assertThat(indices, not(hasItem(SecurityIndexManager.SECURITY_INDEX_NAME)));
         }
@@ -1259,7 +1253,7 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
         {
             IndicesAliasesRequest aliasesRequest = new IndicesAliasesRequest();
             aliasesRequest.addAliasAction(AliasActions.add().alias("security_alias1").index("*"));
-            final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(allAccessUser, IndicesAliasesAction.NAME);
+            final List<String> authorizedIndices = buildAuthorizedIndices(allAccessUser, IndicesAliasesAction.NAME);
             List<String> indices = resolveIndices(aliasesRequest, authorizedIndices).getLocal();
             assertThat(indices, not(hasItem(SecurityIndexManager.SECURITY_INDEX_NAME)));
         }
@@ -1351,7 +1345,7 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
     public void testDynamicPutMappingRequestFromAlias() {
         PutMappingRequest request = new PutMappingRequest(Strings.EMPTY_ARRAY).setConcreteIndex(new Index("foofoo", UUIDs.base64UUID()));
         User user = new User("alias-writer", "alias_read_write");
-        AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, PutMappingAction.NAME);
+        List<String> authorizedIndices = buildAuthorizedIndices(user, PutMappingAction.NAME);
 
         String putMappingIndexOrAlias = IndicesAndAliasesResolver.getPutMappingIndexOrAlias(request, authorizedIndices, metaData);
         assertEquals("barbaz", putMappingIndexOrAlias);
@@ -1366,12 +1360,12 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
     // TODO with the removal of DeleteByQuery is there another way to test resolving a write action?
 
 
-    private AuthorizedIndices buildAuthorizedIndices(User user, String action) {
+    private List<String> buildAuthorizedIndices(User user, String action) {
         PlainActionFuture<Role> rolesListener = new PlainActionFuture<>();
         final Authentication authentication =
             new Authentication(user, new RealmRef("test", "indices-aliases-resolver-tests", "node"), null);
-        authzService.roles(user, authentication, rolesListener);
-        return new AuthorizedIndices(rolesListener.actionGet(), action, metaData);
+        rolesStore.getRoles(user, authentication, rolesListener);
+        return RBACEngine.resolveAuthorizedIndicesFromRole(rolesListener.actionGet(), action, metaData.getAliasAndIndexLookup());
     }
 
     public static IndexMetaData.Builder indexBuilder(String index) {
@@ -1380,7 +1374,7 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
                 .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0));
     }
 
-    private ResolvedIndices resolveIndices(TransportRequest request, AuthorizedIndices authorizedIndices) {
+    private ResolvedIndices resolveIndices(TransportRequest request, List<String> authorizedIndices) {
         return defaultIndicesResolver.resolve(request, this.metaData, authorizedIndices);
     }
 

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/RBACEngineTests.java

@@ -0,0 +1,750 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security.authz;
+
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
+import org.elasticsearch.action.admin.cluster.stats.ClusterStatsAction;
+import org.elasticsearch.action.delete.DeleteAction;
+import org.elasticsearch.action.index.IndexAction;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.collect.MapBuilder;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.license.GetLicenseAction;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.transport.TransportRequest;
+import org.elasticsearch.xpack.core.security.action.user.AuthenticateAction;
+import org.elasticsearch.xpack.core.security.action.user.AuthenticateRequest;
+import org.elasticsearch.xpack.core.security.action.user.AuthenticateRequestBuilder;
+import org.elasticsearch.xpack.core.security.action.user.ChangePasswordAction;
+import org.elasticsearch.xpack.core.security.action.user.ChangePasswordRequest;
+import org.elasticsearch.xpack.core.security.action.user.ChangePasswordRequestBuilder;
+import org.elasticsearch.xpack.core.security.action.user.DeleteUserAction;
+import org.elasticsearch.xpack.core.security.action.user.GetUserPrivilegesResponse;
+import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesRequest;
+import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesResponse;
+import org.elasticsearch.xpack.core.security.action.user.PutUserAction;
+import org.elasticsearch.xpack.core.security.action.user.UserRequest;
+import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authc.esnative.NativeRealmSettings;
+import org.elasticsearch.xpack.core.security.authc.file.FileRealmSettings;
+import org.elasticsearch.xpack.core.security.authc.ldap.LdapRealmSettings;
+import org.elasticsearch.xpack.core.security.authc.pki.PkiRealmSettings;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationInfo;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissions;
+import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsDefinition;
+import org.elasticsearch.xpack.core.security.authz.permission.ResourcePrivileges;
+import org.elasticsearch.xpack.core.security.authz.permission.Role;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
+import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilege;
+import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivileges.ManageApplicationPrivileges;
+import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege;
+import org.elasticsearch.xpack.core.security.authz.privilege.Privilege;
+import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm;
+import org.elasticsearch.xpack.security.authz.RBACEngine.RBACAuthorizationInfo;
+import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore;
+import org.hamcrest.Matchers;
+import org.junit.Before;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
+
+import static java.util.Collections.emptyMap;
+import static org.elasticsearch.common.util.set.Sets.newHashSet;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.emptyIterable;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.iterableWithSize;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.verifyZeroInteractions;
+import static org.mockito.Mockito.when;
+
+public class RBACEngineTests extends ESTestCase {
+
+    private RBACEngine engine;
+
+    @Before
+    public void createEngine() {
+        engine = new RBACEngine(Settings.EMPTY, mock(CompositeRolesStore.class));
+    }
+
+    public void testSameUserPermission() {
+        final User user = new User("joe");
+        final boolean changePasswordRequest = randomBoolean();
+        final TransportRequest request = changePasswordRequest ?
+            new ChangePasswordRequestBuilder(mock(Client.class)).username(user.principal()).request() :
+            new AuthenticateRequestBuilder(mock(Client.class)).username(user.principal()).request();
+        final String action = changePasswordRequest ? ChangePasswordAction.NAME : AuthenticateAction.NAME;
+        final Authentication authentication = mock(Authentication.class);
+        final Authentication.RealmRef authenticatedBy = mock(Authentication.RealmRef.class);
+        when(authentication.getUser()).thenReturn(user);
+        when(authentication.getAuthenticatedBy()).thenReturn(authenticatedBy);
+        when(authenticatedBy.getType())
+            .thenReturn(changePasswordRequest ? randomFrom(ReservedRealm.TYPE, NativeRealmSettings.TYPE) :
+                randomAlphaOfLengthBetween(4, 12));
+
+        assertThat(request, instanceOf(UserRequest.class));
+        assertTrue(engine.checkSameUserPermissions(action, request, authentication));
+    }
+
+    public void testSameUserPermissionDoesNotAllowNonMatchingUsername() {
+        final User authUser = new User("admin", new String[]{"bar"});
+        final User user = new User("joe", null, authUser);
+        final boolean changePasswordRequest = randomBoolean();
+        final String username = randomFrom("", "joe" + randomAlphaOfLengthBetween(1, 5), randomAlphaOfLengthBetween(3, 10));
+        final TransportRequest request = changePasswordRequest ?
+            new ChangePasswordRequestBuilder(mock(Client.class)).username(username).request() :
+            new AuthenticateRequestBuilder(mock(Client.class)).username(username).request();
+        final String action = changePasswordRequest ? ChangePasswordAction.NAME : AuthenticateAction.NAME;
+        final Authentication authentication = mock(Authentication.class);
+        final Authentication.RealmRef authenticatedBy = mock(Authentication.RealmRef.class);
+        when(authentication.getUser()).thenReturn(user);
+        when(authentication.getAuthenticatedBy()).thenReturn(authenticatedBy);
+        when(authenticatedBy.getType())
+            .thenReturn(changePasswordRequest ? randomFrom(ReservedRealm.TYPE, NativeRealmSettings.TYPE) :
+                randomAlphaOfLengthBetween(4, 12));
+
+        assertThat(request, instanceOf(UserRequest.class));
+        assertFalse(engine.checkSameUserPermissions(action, request, authentication));
+
+        when(authentication.getUser()).thenReturn(user);
+        final Authentication.RealmRef lookedUpBy = mock(Authentication.RealmRef.class);
+        when(authentication.getLookedUpBy()).thenReturn(lookedUpBy);
+        when(lookedUpBy.getType())
+            .thenReturn(changePasswordRequest ? randomFrom(ReservedRealm.TYPE, NativeRealmSettings.TYPE) :
+                randomAlphaOfLengthBetween(4, 12));
+        // this should still fail since the username is still different
+        assertFalse(engine.checkSameUserPermissions(action, request, authentication));
+
+        if (request instanceof ChangePasswordRequest) {
+            ((ChangePasswordRequest) request).username("joe");
+        } else {
+            ((AuthenticateRequest) request).username("joe");
+        }
+        assertTrue(engine.checkSameUserPermissions(action, request, authentication));
+    }
+
+    public void testSameUserPermissionDoesNotAllowOtherActions() {
+        final User user = mock(User.class);
+        final TransportRequest request = mock(TransportRequest.class);
+        final String action = randomFrom(PutUserAction.NAME, DeleteUserAction.NAME, ClusterHealthAction.NAME, ClusterStateAction.NAME,
+            ClusterStatsAction.NAME, GetLicenseAction.NAME);
+        final Authentication authentication = mock(Authentication.class);
+        final Authentication.RealmRef authenticatedBy = mock(Authentication.RealmRef.class);
+        final boolean runAs = randomBoolean();
+        when(authentication.getUser()).thenReturn(user);
+        when(user.authenticatedUser()).thenReturn(runAs ? new User("authUser") : user);
+        when(user.isRunAs()).thenReturn(runAs);
+        when(authentication.getAuthenticatedBy()).thenReturn(authenticatedBy);
+        when(authenticatedBy.getType())
+            .thenReturn(randomAlphaOfLengthBetween(4, 12));
+
+        assertFalse(engine.checkSameUserPermissions(action, request, authentication));
+        verifyZeroInteractions(user, request, authentication);
+    }
+
+    public void testSameUserPermissionRunAsChecksAuthenticatedBy() {
+        final User authUser = new User("admin", new String[]{"bar"});
+        final String username = "joe";
+        final User user = new User(username, null, authUser);
+        final boolean changePasswordRequest = randomBoolean();
+        final TransportRequest request = changePasswordRequest ?
+            new ChangePasswordRequestBuilder(mock(Client.class)).username(username).request() :
+            new AuthenticateRequestBuilder(mock(Client.class)).username(username).request();
+        final String action = changePasswordRequest ? ChangePasswordAction.NAME : AuthenticateAction.NAME;
+        final Authentication authentication = mock(Authentication.class);
+        final Authentication.RealmRef authenticatedBy = mock(Authentication.RealmRef.class);
+        final Authentication.RealmRef lookedUpBy = mock(Authentication.RealmRef.class);
+        when(authentication.getUser()).thenReturn(user);
+        when(authentication.getAuthenticatedBy()).thenReturn(authenticatedBy);
+        when(authentication.getLookedUpBy()).thenReturn(lookedUpBy);
+        when(lookedUpBy.getType())
+            .thenReturn(changePasswordRequest ? randomFrom(ReservedRealm.TYPE, NativeRealmSettings.TYPE) :
+                randomAlphaOfLengthBetween(4, 12));
+        assertTrue(engine.checkSameUserPermissions(action, request, authentication));
+
+        when(authentication.getUser()).thenReturn(authUser);
+        assertFalse(engine.checkSameUserPermissions(action, request, authentication));
+    }
+
+    public void testSameUserPermissionDoesNotAllowChangePasswordForOtherRealms() {
+        final User user = new User("joe");
+        final ChangePasswordRequest request = new ChangePasswordRequestBuilder(mock(Client.class)).username(user.principal()).request();
+        final String action = ChangePasswordAction.NAME;
+        final Authentication authentication = mock(Authentication.class);
+        final Authentication.RealmRef authenticatedBy = mock(Authentication.RealmRef.class);
+        when(authentication.getUser()).thenReturn(user);
+        when(authentication.getAuthenticatedBy()).thenReturn(authenticatedBy);
+        when(authenticatedBy.getType()).thenReturn(randomFrom(LdapRealmSettings.LDAP_TYPE, FileRealmSettings.TYPE,
+            LdapRealmSettings.AD_TYPE, PkiRealmSettings.TYPE,
+            randomAlphaOfLengthBetween(4, 12)));
+
+        assertThat(request, instanceOf(UserRequest.class));
+        assertFalse(engine.checkSameUserPermissions(action, request, authentication));
+        verify(authenticatedBy).getType();
+        verify(authentication).getAuthenticatedBy();
+        verify(authentication, times(2)).getUser();
+        verifyNoMoreInteractions(authenticatedBy, authentication);
+    }
+
+    public void testSameUserPermissionDoesNotAllowChangePasswordForLookedUpByOtherRealms() {
+        final User authUser = new User("admin", new String[]{"bar"});
+        final User user = new User("joe", null, authUser);
+        final ChangePasswordRequest request = new ChangePasswordRequestBuilder(mock(Client.class)).username(user.principal()).request();
+        final String action = ChangePasswordAction.NAME;
+        final Authentication authentication = mock(Authentication.class);
+        final Authentication.RealmRef authenticatedBy = mock(Authentication.RealmRef.class);
+        final Authentication.RealmRef lookedUpBy = mock(Authentication.RealmRef.class);
+        when(authentication.getUser()).thenReturn(user);
+        when(authentication.getAuthenticatedBy()).thenReturn(authenticatedBy);
+        when(authentication.getLookedUpBy()).thenReturn(lookedUpBy);
+        when(lookedUpBy.getType()).thenReturn(randomFrom(LdapRealmSettings.LDAP_TYPE, FileRealmSettings.TYPE,
+            LdapRealmSettings.AD_TYPE, PkiRealmSettings.TYPE,
+            randomAlphaOfLengthBetween(4, 12)));
+
+        assertThat(request, instanceOf(UserRequest.class));
+        assertFalse(engine.checkSameUserPermissions(action, request, authentication));
+        verify(authentication).getLookedUpBy();
+        verify(authentication, times(2)).getUser();
+        verify(lookedUpBy).getType();
+        verifyNoMoreInteractions(authentication, lookedUpBy, authenticatedBy);
+    }
+
+    /**
+     * This tests that action names in the request are considered "matched" by the relevant named privilege
+     * (in this case that {@link DeleteAction} and {@link IndexAction} are satisfied by {@link IndexPrivilege#WRITE}).
+     */
+    public void testNamedIndexPrivilegesMatchApplicableActions() throws Exception {
+        User user = new User(randomAlphaOfLengthBetween(4, 12));
+        Authentication authentication = mock(Authentication.class);
+        when(authentication.getUser()).thenReturn(user);
+        Role role = Role.builder("test1")
+            .cluster(Collections.singleton("all"), Collections.emptyList())
+            .add(IndexPrivilege.WRITE, "academy")
+            .build();
+        RBACAuthorizationInfo authzInfo = new RBACAuthorizationInfo(role, null);
+
+        final HasPrivilegesRequest request = new HasPrivilegesRequest();
+        request.username(user.principal());
+        request.clusterPrivileges(ClusterHealthAction.NAME);
+        request.indexPrivileges(RoleDescriptor.IndicesPrivileges.builder()
+            .indices("academy")
+            .privileges(DeleteAction.NAME, IndexAction.NAME)
+            .build());
+        request.applicationPrivileges(new RoleDescriptor.ApplicationResourcePrivileges[0]);
+
+        final PlainActionFuture<HasPrivilegesResponse> future = new PlainActionFuture<>();
+        engine.checkPrivileges(authentication, authzInfo, request, Collections.emptyList(), future);
+
+        final HasPrivilegesResponse response = future.get();
+        assertThat(response, notNullValue());
+        assertThat(response.getUsername(), is(user.principal()));
+        assertThat(response.isCompleteMatch(), is(true));
+
+        assertThat(response.getClusterPrivileges().size(), equalTo(1));
+        assertThat(response.getClusterPrivileges().get(ClusterHealthAction.NAME), equalTo(true));
+
+        assertThat(response.getIndexPrivileges(), Matchers.iterableWithSize(1));
+        final ResourcePrivileges result = response.getIndexPrivileges().iterator().next();
+        assertThat(result.getResource(), equalTo("academy"));
+        assertThat(result.getPrivileges().size(), equalTo(2));
+        assertThat(result.getPrivileges().get(DeleteAction.NAME), equalTo(true));
+        assertThat(result.getPrivileges().get(IndexAction.NAME), equalTo(true));
+    }
+
+    /**
+     * This tests that the action responds correctly when the user/role has some, but not all
+     * of the privileges being checked.
+     */
+    public void testMatchSubsetOfPrivileges() throws Exception {
+        User user = new User(randomAlphaOfLengthBetween(4, 12));
+        Authentication authentication = mock(Authentication.class);
+        when(authentication.getUser()).thenReturn(user);
+        Role role = Role.builder("test2")
+            .cluster(ClusterPrivilege.MONITOR)
+            .add(IndexPrivilege.INDEX, "academy")
+            .add(IndexPrivilege.WRITE, "initiative")
+            .build();
+        RBACAuthorizationInfo authzInfo = new RBACAuthorizationInfo(role, null);
+
+        final HasPrivilegesRequest request = new HasPrivilegesRequest();
+        request.username(user.principal());
+        request.clusterPrivileges("monitor", "manage");
+        request.indexPrivileges(RoleDescriptor.IndicesPrivileges.builder()
+            .indices("academy", "initiative", "school")
+            .privileges("delete", "index", "manage")
+            .build());
+        request.applicationPrivileges(new RoleDescriptor.ApplicationResourcePrivileges[0]);
+        final PlainActionFuture<HasPrivilegesResponse> future = new PlainActionFuture<>();
+        engine.checkPrivileges(authentication, authzInfo, request, Collections.emptyList(), future);
+
+        final HasPrivilegesResponse response = future.get();
+        assertThat(response, notNullValue());
+        assertThat(response.getUsername(), is(user.principal()));
+        assertThat(response.isCompleteMatch(), is(false));
+        assertThat(response.getClusterPrivileges().size(), equalTo(2));
+        assertThat(response.getClusterPrivileges().get("monitor"), equalTo(true));
+        assertThat(response.getClusterPrivileges().get("manage"), equalTo(false));
+        assertThat(response.getIndexPrivileges(), Matchers.iterableWithSize(3));
+
+        final Iterator<ResourcePrivileges> indexPrivilegesIterator = response.getIndexPrivileges().iterator();
+        final ResourcePrivileges academy = indexPrivilegesIterator.next();
+        final ResourcePrivileges initiative = indexPrivilegesIterator.next();
+        final ResourcePrivileges school = indexPrivilegesIterator.next();
+
+        assertThat(academy.getResource(), equalTo("academy"));
+        assertThat(academy.getPrivileges().size(), equalTo(3));
+        assertThat(academy.getPrivileges().get("index"), equalTo(true)); // explicit
+        assertThat(academy.getPrivileges().get("delete"), equalTo(false));
+        assertThat(academy.getPrivileges().get("manage"), equalTo(false));
+
+        assertThat(initiative.getResource(), equalTo("initiative"));
+        assertThat(initiative.getPrivileges().size(), equalTo(3));
+        assertThat(initiative.getPrivileges().get("index"), equalTo(true)); // implied by write
+        assertThat(initiative.getPrivileges().get("delete"), equalTo(true)); // implied by write
+        assertThat(initiative.getPrivileges().get("manage"), equalTo(false));
+
+        assertThat(school.getResource(), equalTo("school"));
+        assertThat(school.getPrivileges().size(), equalTo(3));
+        assertThat(school.getPrivileges().get("index"), equalTo(false));
+        assertThat(school.getPrivileges().get("delete"), equalTo(false));
+        assertThat(school.getPrivileges().get("manage"), equalTo(false));
+    }
+
+    /**
+     * This tests that the action responds correctly when the user/role has none
+     * of the privileges being checked.
+     */
+    public void testMatchNothing() throws Exception {
+        User user = new User(randomAlphaOfLengthBetween(4, 12));
+        Authentication authentication = mock(Authentication.class);
+        when(authentication.getUser()).thenReturn(user);
+        Role role = Role.builder("test3")
+            .cluster(ClusterPrivilege.MONITOR)
+            .build();
+        RBACAuthorizationInfo authzInfo = new RBACAuthorizationInfo(role, null);
+
+        final HasPrivilegesResponse response = hasPrivileges(RoleDescriptor.IndicesPrivileges.builder()
+            .indices("academy")
+            .privileges("read", "write")
+            .build(),
+            authentication, authzInfo, Collections.emptyList(), Strings.EMPTY_ARRAY);
+        assertThat(response.getUsername(), is(user.principal()));
+        assertThat(response.isCompleteMatch(), is(false));
+        assertThat(response.getIndexPrivileges(), Matchers.iterableWithSize(1));
+        final ResourcePrivileges result = response.getIndexPrivileges().iterator().next();
+        assertThat(result.getResource(), equalTo("academy"));
+        assertThat(result.getPrivileges().size(), equalTo(2));
+        assertThat(result.getPrivileges().get("read"), equalTo(false));
+        assertThat(result.getPrivileges().get("write"), equalTo(false));
+    }
+
+    /**
+     * Wildcards in the request are treated as
+     * <em>does the user have ___ privilege on every possible index that matches this pattern?</em>
+     * Or, expressed differently,
+     * <em>does the user have ___ privilege on a wildcard that covers (is a superset of) this pattern?</em>
+     */
+    public void testWildcardHandling() throws Exception {
+        List<ApplicationPrivilegeDescriptor> privs = new ArrayList<>();
+        final ApplicationPrivilege kibanaRead = defineApplicationPrivilege(privs, "kibana", "read",
+            "data:read/*", "action:login", "action:view/dashboard");
+        final ApplicationPrivilege kibanaWrite = defineApplicationPrivilege(privs, "kibana", "write",
+            "data:write/*", "action:login", "action:view/dashboard");
+        final ApplicationPrivilege kibanaAdmin = defineApplicationPrivilege(privs, "kibana", "admin",
+            "action:login", "action:manage/*");
+        final ApplicationPrivilege kibanaViewSpace = defineApplicationPrivilege(privs, "kibana", "view-space",
+            "action:login", "space:view/*");
+        User user = new User(randomAlphaOfLengthBetween(4, 12));
+        Authentication authentication = mock(Authentication.class);
+        when(authentication.getUser()).thenReturn(user);
+        Role role = Role.builder("test3")
+            .add(IndexPrivilege.ALL, "logstash-*", "foo?")
+            .add(IndexPrivilege.READ, "abc*")
+            .add(IndexPrivilege.WRITE, "*xyz")
+            .addApplicationPrivilege(kibanaRead, Collections.singleton("*"))
+            .addApplicationPrivilege(kibanaViewSpace, newHashSet("space/engineering/*", "space/builds"))
+            .build();
+        RBACAuthorizationInfo authzInfo = new RBACAuthorizationInfo(role, null);
+
+        final HasPrivilegesRequest request = new HasPrivilegesRequest();
+        request.username(user.principal());
+        request.clusterPrivileges(Strings.EMPTY_ARRAY);
+        request.indexPrivileges(
+            RoleDescriptor.IndicesPrivileges.builder()
+                .indices("logstash-2016-*")
+                .privileges("write") // Yes, because (ALL,"logstash-*")
+                .build(),
+            RoleDescriptor.IndicesPrivileges.builder()
+                .indices("logstash-*")
+                .privileges("read") // Yes, because (ALL,"logstash-*")
+                .build(),
+            RoleDescriptor.IndicesPrivileges.builder()
+                .indices("log*")
+                .privileges("manage") // No, because "log*" includes indices that "logstash-*" does not
+                .build(),
+            RoleDescriptor.IndicesPrivileges.builder()
+                .indices("foo*", "foo?")
+                .privileges("read") // Yes, "foo?", but not "foo*", because "foo*" > "foo?"
+                .build(),
+            RoleDescriptor.IndicesPrivileges.builder()
+                .indices("abcd*")
+                .privileges("read", "write") // read = Yes, because (READ, "abc*"), write = No
+                .build(),
+            RoleDescriptor.IndicesPrivileges.builder()
+                .indices("abc*xyz")
+                .privileges("read", "write", "manage") // read = Yes ( READ "abc*"), write = Yes (WRITE, "*xyz"), manage = No
+                .build(),
+            RoleDescriptor.IndicesPrivileges.builder()
+                .indices("a*xyz")
+                .privileges("read", "write", "manage") // read = No, write = Yes (WRITE, "*xyz"), manage = No
+                .build()
+        );
+
+        request.applicationPrivileges(
+            RoleDescriptor.ApplicationResourcePrivileges.builder()
+                .resources("*")
+                .application("kibana")
+                .privileges(Sets.union(kibanaRead.name(), kibanaWrite.name())) // read = Yes, write = No
+                .build(),
+            RoleDescriptor.ApplicationResourcePrivileges.builder()
+                .resources("space/engineering/project-*", "space/*") // project-* = Yes, space/* = Not
+                .application("kibana")
+                .privileges("space:view/dashboard")
+                .build()
+        );
+
+        final PlainActionFuture<HasPrivilegesResponse> future = new PlainActionFuture<>();
+        engine.checkPrivileges(authentication, authzInfo, request, privs, future);
+
+        final HasPrivilegesResponse response = future.get();
+        assertThat(response, notNullValue());
+        assertThat(response.getUsername(), is(user.principal()));
+        assertThat(response.isCompleteMatch(), is(false));
+        assertThat(response.getIndexPrivileges(), Matchers.iterableWithSize(8));
+        assertThat(response.getIndexPrivileges(), containsInAnyOrder(
+            ResourcePrivileges.builder("logstash-2016-*").addPrivileges(Collections.singletonMap("write", true)).build(),
+            ResourcePrivileges.builder("logstash-*").addPrivileges(Collections.singletonMap("read", true)).build(),
+            ResourcePrivileges.builder("log*").addPrivileges(Collections.singletonMap("manage", false)).build(),
+            ResourcePrivileges.builder("foo?").addPrivileges(Collections.singletonMap("read", true)).build(),
+            ResourcePrivileges.builder("foo*").addPrivileges(Collections.singletonMap("read", false)).build(),
+            ResourcePrivileges.builder("abcd*").addPrivileges(mapBuilder().put("read", true).put("write", false).map()).build(),
+            ResourcePrivileges.builder("abc*xyz")
+                .addPrivileges(mapBuilder().put("read", true).put("write", true).put("manage", false).map()).build(),
+            ResourcePrivileges.builder("a*xyz")
+                .addPrivileges(mapBuilder().put("read", false).put("write", true).put("manage", false).map()).build()
+        ));
+        assertThat(response.getApplicationPrivileges().entrySet(), Matchers.iterableWithSize(1));
+        final Set<ResourcePrivileges> kibanaPrivileges = response.getApplicationPrivileges().get("kibana");
+        assertThat(kibanaPrivileges, Matchers.iterableWithSize(3));
+        assertThat(Strings.collectionToCommaDelimitedString(kibanaPrivileges), kibanaPrivileges, containsInAnyOrder(
+            ResourcePrivileges.builder("*").addPrivileges(mapBuilder().put("read", true).put("write", false).map()).build(),
+            ResourcePrivileges.builder("space/engineering/project-*")
+                .addPrivileges(Collections.singletonMap("space:view/dashboard", true)).build(),
+            ResourcePrivileges.builder("space/*").addPrivileges(Collections.singletonMap("space:view/dashboard", false)).build()
+        ));
+    }
+
+    public void testCheckingIndexPermissionsDefinedOnDifferentPatterns() throws Exception {
+        User user = new User(randomAlphaOfLengthBetween(4, 12));
+        Authentication authentication = mock(Authentication.class);
+        when(authentication.getUser()).thenReturn(user);
+        Role role = Role.builder("test-write")
+            .add(IndexPrivilege.INDEX, "apache-*")
+            .add(IndexPrivilege.DELETE, "apache-2016-*")
+            .build();
+        RBACAuthorizationInfo authzInfo = new RBACAuthorizationInfo(role, null);
+
+        final HasPrivilegesResponse response = hasPrivileges(RoleDescriptor.IndicesPrivileges.builder()
+            .indices("apache-2016-12", "apache-2017-01")
+            .privileges("index", "delete")
+            .build(), authentication, authzInfo, Collections.emptyList(), Strings.EMPTY_ARRAY);
+        assertThat(response.isCompleteMatch(), is(false));
+        assertThat(response.getIndexPrivileges(), Matchers.iterableWithSize(2));
+        assertThat(response.getIndexPrivileges(), containsInAnyOrder(
+            ResourcePrivileges.builder("apache-2016-12")
+                .addPrivileges(MapBuilder.newMapBuilder(new LinkedHashMap<String, Boolean>())
+                    .put("index", true).put("delete", true).map()).build(),
+            ResourcePrivileges.builder("apache-2017-01")
+                .addPrivileges(MapBuilder.newMapBuilder(new LinkedHashMap<String, Boolean>())
+                    .put("index", true).put("delete", false).map()).build()
+        ));
+    }
+
+    public void testCheckingApplicationPrivilegesOnDifferentApplicationsAndResources() throws Exception {
+        List<ApplicationPrivilegeDescriptor> privs = new ArrayList<>();
+        final ApplicationPrivilege app1Read = defineApplicationPrivilege(privs, "app1", "read", "data:read/*");
+        final ApplicationPrivilege app1Write = defineApplicationPrivilege(privs, "app1", "write", "data:write/*");
+        final ApplicationPrivilege app1All = defineApplicationPrivilege(privs, "app1", "all", "*");
+        final ApplicationPrivilege app2Read = defineApplicationPrivilege(privs, "app2", "read", "data:read/*");
+        final ApplicationPrivilege app2Write = defineApplicationPrivilege(privs, "app2", "write", "data:write/*");
+        final ApplicationPrivilege app2All = defineApplicationPrivilege(privs, "app2", "all", "*");
+
+        User user = new User(randomAlphaOfLengthBetween(4, 12));
+        Authentication authentication = mock(Authentication.class);
+        when(authentication.getUser()).thenReturn(user);
+        Role role = Role.builder("test-role")
+            .addApplicationPrivilege(app1Read, Collections.singleton("foo/*"))
+            .addApplicationPrivilege(app1All, Collections.singleton("foo/bar/baz"))
+            .addApplicationPrivilege(app2Read, Collections.singleton("foo/bar/*"))
+            .addApplicationPrivilege(app2Write, Collections.singleton("*/bar/*"))
+            .build();
+        RBACAuthorizationInfo authzInfo = new RBACAuthorizationInfo(role, null);
+
+        final HasPrivilegesResponse response = hasPrivileges(new RoleDescriptor.IndicesPrivileges[0],
+            new RoleDescriptor.ApplicationResourcePrivileges[]{
+                RoleDescriptor.ApplicationResourcePrivileges.builder()
+                    .application("app1")
+                    .resources("foo/1", "foo/bar/2", "foo/bar/baz", "baz/bar/foo")
+                    .privileges("read", "write", "all")
+                    .build(),
+                RoleDescriptor.ApplicationResourcePrivileges.builder()
+                    .application("app2")
+                    .resources("foo/1", "foo/bar/2", "foo/bar/baz", "baz/bar/foo")
+                    .privileges("read", "write", "all")
+                    .build()
+            }, authentication, authzInfo, privs, Strings.EMPTY_ARRAY);
+
+        assertThat(response.isCompleteMatch(), is(false));
+        assertThat(response.getIndexPrivileges(), Matchers.emptyIterable());
+        assertThat(response.getApplicationPrivileges().entrySet(), Matchers.iterableWithSize(2));
+        final Set<ResourcePrivileges> app1 = response.getApplicationPrivileges().get("app1");
+        assertThat(app1, Matchers.iterableWithSize(4));
+        assertThat(Strings.collectionToCommaDelimitedString(app1), app1, containsInAnyOrder(
+            ResourcePrivileges.builder("foo/1").addPrivileges(MapBuilder.newMapBuilder(new LinkedHashMap<String, Boolean>())
+                .put("read", true).put("write", false).put("all", false).map()).build(),
+            ResourcePrivileges.builder("foo/bar/2").addPrivileges(MapBuilder.newMapBuilder(new LinkedHashMap<String, Boolean>())
+                .put("read", true).put("write", false).put("all", false).map()).build(),
+            ResourcePrivileges.builder("foo/bar/baz").addPrivileges(MapBuilder.newMapBuilder(new LinkedHashMap<String, Boolean>())
+                .put("read", true).put("write", true).put("all", true).map()).build(),
+            ResourcePrivileges.builder("baz/bar/foo").addPrivileges(MapBuilder.newMapBuilder(new LinkedHashMap<String, Boolean>())
+                .put("read", false).put("write", false).put("all", false).map()).build()
+        ));
+        final Set<ResourcePrivileges> app2 = response.getApplicationPrivileges().get("app2");
+        assertThat(app2, Matchers.iterableWithSize(4));
+        assertThat(Strings.collectionToCommaDelimitedString(app2), app2, containsInAnyOrder(
+            ResourcePrivileges.builder("foo/1").addPrivileges(MapBuilder.newMapBuilder(new LinkedHashMap<String, Boolean>())
+                .put("read", false).put("write", false).put("all", false).map()).build(),
+            ResourcePrivileges.builder("foo/bar/2").addPrivileges(MapBuilder.newMapBuilder(new LinkedHashMap<String, Boolean>())
+                .put("read", true).put("write", true).put("all", false).map()).build(),
+            ResourcePrivileges.builder("foo/bar/baz").addPrivileges(MapBuilder.newMapBuilder(new LinkedHashMap<String, Boolean>())
+                .put("read", true).put("write", true).put("all", false).map()).build(),
+            ResourcePrivileges.builder("baz/bar/foo").addPrivileges(MapBuilder.newMapBuilder(new LinkedHashMap<String, Boolean>())
+                .put("read", false).put("write", true).put("all", false).map()).build()
+        ));
+    }
+
+    public void testCheckingApplicationPrivilegesWithComplexNames() throws Exception {
+        final String appName = randomAlphaOfLength(1).toLowerCase(Locale.ROOT) + randomAlphaOfLengthBetween(3, 10);
+        final String action1 = randomAlphaOfLength(1).toLowerCase(Locale.ROOT) + randomAlphaOfLengthBetween(2, 5);
+        final String action2 = randomAlphaOfLength(1).toLowerCase(Locale.ROOT) + randomAlphaOfLengthBetween(6, 9);
+
+        final List<ApplicationPrivilegeDescriptor> privs = new ArrayList<>();
+        final ApplicationPrivilege priv1 = defineApplicationPrivilege(privs, appName, action1, "DATA:read/*", "ACTION:" + action1);
+        final ApplicationPrivilege priv2 = defineApplicationPrivilege(privs, appName, action2, "DATA:read/*", "ACTION:" + action2);
+
+        User user = new User(randomAlphaOfLengthBetween(4, 12));
+        Authentication authentication = mock(Authentication.class);
+        when(authentication.getUser()).thenReturn(user);
+        Role role = Role.builder("test-write")
+            .addApplicationPrivilege(priv1, Collections.singleton("user/*/name"))
+            .build();
+        RBACAuthorizationInfo authzInfo = new RBACAuthorizationInfo(role, null);
+
+        final HasPrivilegesResponse response = hasPrivileges(
+            new RoleDescriptor.IndicesPrivileges[0],
+            new RoleDescriptor.ApplicationResourcePrivileges[]{
+                RoleDescriptor.ApplicationResourcePrivileges.builder()
+                    .application(appName)
+                    .resources("user/hawkeye/name")
+                    .privileges("DATA:read/user/*", "ACTION:" + action1, "ACTION:" + action2, action1, action2)
+                    .build()
+            }, authentication, authzInfo, privs, "monitor");
+        assertThat(response.isCompleteMatch(), is(false));
+        assertThat(response.getApplicationPrivileges().keySet(), containsInAnyOrder(appName));
+        assertThat(response.getApplicationPrivileges().get(appName), iterableWithSize(1));
+        assertThat(response.getApplicationPrivileges().get(appName), containsInAnyOrder(
+            ResourcePrivileges.builder("user/hawkeye/name").addPrivileges(MapBuilder.newMapBuilder(new LinkedHashMap<String, Boolean>())
+                .put("DATA:read/user/*", true)
+                .put("ACTION:" + action1, true)
+                .put("ACTION:" + action2, false)
+                .put(action1, true)
+                .put(action2, false)
+                .map()).build()
+        ));
+    }
+
+    public void testIsCompleteMatch() throws Exception {
+        final List<ApplicationPrivilegeDescriptor> privs = new ArrayList<>();
+        final ApplicationPrivilege kibanaRead = defineApplicationPrivilege(privs, "kibana", "read", "data:read/*");
+        final ApplicationPrivilege kibanaWrite = defineApplicationPrivilege(privs, "kibana", "write", "data:write/*");
+        User user = new User(randomAlphaOfLengthBetween(4, 12));
+        Authentication authentication = mock(Authentication.class);
+        when(authentication.getUser()).thenReturn(user);
+        Role role = Role.builder("test-write")
+            .cluster(ClusterPrivilege.MONITOR)
+            .add(IndexPrivilege.READ, "read-*")
+            .add(IndexPrivilege.ALL, "all-*")
+            .addApplicationPrivilege(kibanaRead, Collections.singleton("*"))
+            .build();
+        RBACAuthorizationInfo authzInfo = new RBACAuthorizationInfo(role, null);
+
+
+        assertThat(hasPrivileges(
+            indexPrivileges("read", "read-123", "read-456", "all-999"), authentication, authzInfo, privs, "monitor").isCompleteMatch(),
+            is(true));
+        assertThat(hasPrivileges(
+            indexPrivileges("read", "read-123", "read-456", "all-999"), authentication, authzInfo, privs, "manage").isCompleteMatch(),
+            is(false));
+        assertThat(hasPrivileges(
+            indexPrivileges("write", "read-123", "read-456", "all-999"), authentication, authzInfo, privs, "monitor").isCompleteMatch(),
+            is(false));
+        assertThat(hasPrivileges(
+            indexPrivileges("write", "read-123", "read-456", "all-999"), authentication, authzInfo, privs, "manage").isCompleteMatch(),
+            is(false));
+        assertThat(hasPrivileges(
+            new RoleDescriptor.IndicesPrivileges[]{
+                RoleDescriptor.IndicesPrivileges.builder()
+                    .indices("read-a")
+                    .privileges("read")
+                    .build(),
+                RoleDescriptor.IndicesPrivileges.builder()
+                    .indices("all-b")
+                    .privileges("read", "write")
+                    .build()
+            },
+            new RoleDescriptor.ApplicationResourcePrivileges[]{
+                RoleDescriptor.ApplicationResourcePrivileges.builder()
+                    .application("kibana")
+                    .resources("*")
+                    .privileges("read")
+                    .build()
+            }, authentication, authzInfo, privs, "monitor").isCompleteMatch(), is(true));
+        assertThat(hasPrivileges(
+            new RoleDescriptor.IndicesPrivileges[]{indexPrivileges("read", "read-123", "read-456", "all-999")},
+            new RoleDescriptor.ApplicationResourcePrivileges[]{
+                RoleDescriptor.ApplicationResourcePrivileges.builder()
+                    .application("kibana").resources("*").privileges("read").build(),
+                RoleDescriptor.ApplicationResourcePrivileges.builder()
+                    .application("kibana").resources("*").privileges("write").build()
+            }, authentication, authzInfo, privs, "monitor").isCompleteMatch(), is(false));
+    }
+
+    public void testBuildUserPrivilegeResponse() {
+        final ManageApplicationPrivileges manageApplicationPrivileges = new ManageApplicationPrivileges(Sets.newHashSet("app01", "app02"));
+        final BytesArray query = new BytesArray("{\"term\":{\"public\":true}}");
+        final Role role = Role.builder("test", "role")
+            .cluster(Sets.newHashSet("monitor", "manage_watcher"), Collections.singleton(manageApplicationPrivileges))
+            .add(IndexPrivilege.get(Sets.newHashSet("read", "write")), "index-1")
+            .add(IndexPrivilege.ALL, "index-2", "index-3")
+            .add(
+                new FieldPermissions(new FieldPermissionsDefinition(new String[]{ "public.*" }, new String[0])),
+                Collections.singleton(query),
+                IndexPrivilege.READ, randomBoolean(), "index-4", "index-5")
+            .addApplicationPrivilege(new ApplicationPrivilege("app01", "read", "data:read"), Collections.singleton("*"))
+            .runAs(new Privilege(Sets.newHashSet("user01", "user02"), "user01", "user02"))
+            .build();
+
+        final GetUserPrivilegesResponse response = engine.buildUserPrivilegesResponseObject(role);
+
+        assertThat(response.getClusterPrivileges(), containsInAnyOrder("monitor", "manage_watcher"));
+        assertThat(response.getConditionalClusterPrivileges(), containsInAnyOrder(manageApplicationPrivileges));
+
+        assertThat(response.getIndexPrivileges(), iterableWithSize(3));
+        final GetUserPrivilegesResponse.Indices index1 = findIndexPrivilege(response.getIndexPrivileges(), "index-1");
+        assertThat(index1.getIndices(), containsInAnyOrder("index-1"));
+        assertThat(index1.getPrivileges(), containsInAnyOrder("read", "write"));
+        assertThat(index1.getFieldSecurity(), emptyIterable());
+        assertThat(index1.getQueries(), emptyIterable());
+        final GetUserPrivilegesResponse.Indices index2 = findIndexPrivilege(response.getIndexPrivileges(), "index-2");
+        assertThat(index2.getIndices(), containsInAnyOrder("index-2", "index-3"));
+        assertThat(index2.getPrivileges(), containsInAnyOrder("all"));
+        assertThat(index2.getFieldSecurity(), emptyIterable());
+        assertThat(index2.getQueries(), emptyIterable());
+        final GetUserPrivilegesResponse.Indices index4 = findIndexPrivilege(response.getIndexPrivileges(), "index-4");
+        assertThat(index4.getIndices(), containsInAnyOrder("index-4", "index-5"));
+        assertThat(index4.getPrivileges(), containsInAnyOrder("read"));
+        assertThat(index4.getFieldSecurity(), containsInAnyOrder(
+            new FieldPermissionsDefinition.FieldGrantExcludeGroup(new String[]{ "public.*" }, new String[0])));
+        assertThat(index4.getQueries(), containsInAnyOrder(query));
+
+        assertThat(response.getApplicationPrivileges(), containsInAnyOrder(
+            RoleDescriptor.ApplicationResourcePrivileges.builder().application("app01").privileges("read").resources("*").build())
+        );
+
+        assertThat(response.getRunAs(), containsInAnyOrder("user01", "user02"));
+    }
+
+    private GetUserPrivilegesResponse.Indices findIndexPrivilege(Set<GetUserPrivilegesResponse.Indices> indices, String name) {
+        return indices.stream().filter(i -> i.getIndices().contains(name)).findFirst().get();
+    }
+
+    private RoleDescriptor.IndicesPrivileges indexPrivileges(String priv, String... indices) {
+        return RoleDescriptor.IndicesPrivileges.builder()
+            .indices(indices)
+            .privileges(priv)
+            .build();
+    }
+
+    private ApplicationPrivilege defineApplicationPrivilege(List<ApplicationPrivilegeDescriptor> privs, String app, String name,
+                                                            String ... actions) {
+        privs.add(new ApplicationPrivilegeDescriptor(app, name, newHashSet(actions), emptyMap()));
+        return new ApplicationPrivilege(app, name, actions);
+    }
+
+    private HasPrivilegesResponse hasPrivileges(RoleDescriptor.IndicesPrivileges indicesPrivileges, Authentication authentication,
+                                                AuthorizationInfo authorizationInfo,
+                                                List<ApplicationPrivilegeDescriptor> applicationPrivilegeDescriptors,
+                                                String... clusterPrivileges) throws Exception {
+        return hasPrivileges(
+            new RoleDescriptor.IndicesPrivileges[]{indicesPrivileges},
+            new RoleDescriptor.ApplicationResourcePrivileges[0],
+            authentication, authorizationInfo, applicationPrivilegeDescriptors,
+            clusterPrivileges
+        );
+    }
+
+    private HasPrivilegesResponse hasPrivileges(RoleDescriptor.IndicesPrivileges[] indicesPrivileges,
+                                                RoleDescriptor.ApplicationResourcePrivileges[] appPrivileges,
+                                                Authentication authentication,
+                                                AuthorizationInfo authorizationInfo,
+                                                List<ApplicationPrivilegeDescriptor> applicationPrivilegeDescriptors,
+                                                String... clusterPrivileges) throws Exception {
+        final HasPrivilegesRequest request = new HasPrivilegesRequest();
+        request.username(authentication.getUser().principal());
+        request.clusterPrivileges(clusterPrivileges);
+        request.indexPrivileges(indicesPrivileges);
+        request.applicationPrivileges(appPrivileges);
+        final PlainActionFuture<HasPrivilegesResponse> future = new PlainActionFuture<>();
+        engine.checkPrivileges(authentication, authorizationInfo, request, applicationPrivilegeDescriptors, future);
+        final HasPrivilegesResponse response = future.get();
+        assertThat(response, notNullValue());
+        return response;
+    }
+
+    private static MapBuilder<String, Boolean> mapBuilder() {
+        return MapBuilder.newMapBuilder();
+    }
+}

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java

@@ -24,11 +24,17 @@ import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationField;
 import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.security.audit.AuditTrailService;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationInfo;
+
+import java.util.Collections;
 
 import static org.elasticsearch.mock.orig.Mockito.verifyNoMoreInteractions;
+import static org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail.PRINCIPAL_ROLES_FIELD_NAME;
+import static org.elasticsearch.xpack.security.authz.AuthorizationService.AUTHORIZATION_INFO_KEY;
 import static org.elasticsearch.xpack.security.authz.AuthorizationService.ORIGINATING_ACTION_KEY;
-import static org.elasticsearch.xpack.security.authz.AuthorizationService.ROLE_NAMES_KEY;
+import static org.elasticsearch.xpack.security.authz.AuthorizationServiceTests.authzInfoRoles;
 import static org.elasticsearch.xpack.security.authz.SecuritySearchOperationListener.ensureAuthenticatedUserIsSame;
+import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
@@ -112,13 +118,15 @@ public class SecuritySearchOperationListenerTests extends ESTestCase {
             Authentication authentication = new Authentication(new User("test", "role"), new RealmRef(realmName, type, nodeName), null);
             authentication.writeToContext(threadContext);
             threadContext.putTransient(ORIGINATING_ACTION_KEY, "action");
-            threadContext.putTransient(ROLE_NAMES_KEY, authentication.getUser().roles());
+            threadContext.putTransient(AUTHORIZATION_INFO_KEY,
+                (AuthorizationInfo) () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, authentication.getUser().roles()));
             final InternalScrollSearchRequest request = new InternalScrollSearchRequest();
             SearchContextMissingException expected =
                     expectThrows(SearchContextMissingException.class, () -> listener.validateSearchContext(testSearchContext, request));
             assertEquals(testSearchContext.id(), expected.id());
             verify(licenseState, times(3)).isAuthAllowed();
-            verify(auditTrailService).accessDenied(null, authentication, "action", request, authentication.getUser().roles());
+            verify(auditTrailService).accessDenied(eq(null), eq(authentication), eq("action"), eq(request),
+                authzInfoRoles(authentication.getUser().roles()));
         }
 
         // another user running as the original user
@@ -146,13 +154,15 @@ public class SecuritySearchOperationListenerTests extends ESTestCase {
                     new Authentication(new User("authenticated", "runas"), new RealmRef(realmName, type, nodeName), null);
             authentication.writeToContext(threadContext);
             threadContext.putTransient(ORIGINATING_ACTION_KEY, "action");
-            threadContext.putTransient(ROLE_NAMES_KEY, authentication.getUser().roles());
+            threadContext.putTransient(AUTHORIZATION_INFO_KEY,
+                (AuthorizationInfo) () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, authentication.getUser().roles()));
             final InternalScrollSearchRequest request = new InternalScrollSearchRequest();
             SearchContextMissingException expected =
                     expectThrows(SearchContextMissingException.class, () -> listener.validateSearchContext(testSearchContext, request));
             assertEquals(testSearchContext.id(), expected.id());
             verify(licenseState, times(5)).isAuthAllowed();
-            verify(auditTrailService).accessDenied(null, authentication, "action", request, authentication.getUser().roles());
+            verify(auditTrailService).accessDenied(eq(null), eq(authentication), eq("action"), eq(request),
+                authzInfoRoles(authentication.getUser().roles()));
         }
     }
 
@@ -166,21 +176,24 @@ public class SecuritySearchOperationListenerTests extends ESTestCase {
         AuditTrailService auditTrail = mock(AuditTrailService.class);
 
         final String auditId = randomAlphaOfLengthBetween(8, 20);
-        ensureAuthenticatedUserIsSame(original, current, auditTrail, id, action, request, auditId, original.getUser().roles());
+        ensureAuthenticatedUserIsSame(original, current, auditTrail, id, action, request, auditId,
+            () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, original.getUser().roles()));
         verifyZeroInteractions(auditTrail);
 
         // original user being run as
         User user = new User(new User("test", "role"), new User("authenticated", "runas"));
         current = new Authentication(user, new RealmRef("realm", "file", "node"),
                 new RealmRef(randomAlphaOfLengthBetween(1, 16), "file", "node"));
-        ensureAuthenticatedUserIsSame(original, current, auditTrail, id, action, request, auditId, original.getUser().roles());
+        ensureAuthenticatedUserIsSame(original, current, auditTrail, id, action, request, auditId,
+            () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, original.getUser().roles()));
         verifyZeroInteractions(auditTrail);
 
         // both user are run as
         current = new Authentication(user, new RealmRef("realm", "file", "node"),
                 new RealmRef(randomAlphaOfLengthBetween(1, 16), "file", "node"));
         Authentication runAs = current;
-        ensureAuthenticatedUserIsSame(runAs, current, auditTrail, id, action, request, auditId, original.getUser().roles());
+        ensureAuthenticatedUserIsSame(runAs, current, auditTrail, id, action, request, auditId,
+            () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, original.getUser().roles()));
         verifyZeroInteractions(auditTrail);
 
         // different authenticated by type
@@ -188,36 +201,39 @@ public class SecuritySearchOperationListenerTests extends ESTestCase {
                 new Authentication(new User("test", "role"), new RealmRef("realm", randomAlphaOfLength(5), "node"), null);
         SearchContextMissingException e = expectThrows(SearchContextMissingException.class,
                 () -> ensureAuthenticatedUserIsSame(original, differentRealmType, auditTrail, id, action, request, auditId,
-                        original.getUser().roles()));
+                    () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, original.getUser().roles())));
         assertEquals(id, e.id());
-        verify(auditTrail).accessDenied(auditId, differentRealmType, action, request, original.getUser().roles());
+        verify(auditTrail).accessDenied(eq(auditId), eq(differentRealmType), eq(action), eq(request),
+            authzInfoRoles(original.getUser().roles()));
 
         // wrong user
         Authentication differentUser =
                 new Authentication(new User("test2", "role"), new RealmRef("realm", "realm", "node"), null);
         e = expectThrows(SearchContextMissingException.class,
                 () -> ensureAuthenticatedUserIsSame(original, differentUser, auditTrail, id, action, request, auditId,
-                    original.getUser().roles()));
+                    () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, original.getUser().roles())));
         assertEquals(id, e.id());
-        verify(auditTrail).accessDenied(auditId, differentUser, action, request, original.getUser().roles());
+        verify(auditTrail).accessDenied(eq(auditId), eq(differentUser), eq(action), eq(request),
+            authzInfoRoles(original.getUser().roles()));
 
         // run as different user
         Authentication diffRunAs = new Authentication(new User(new User("test2", "role"), new User("authenticated", "runas")),
                 new RealmRef("realm", "file", "node1"), new RealmRef("realm", "file", "node1"));
         e = expectThrows(SearchContextMissingException.class,
                 () -> ensureAuthenticatedUserIsSame(original, diffRunAs, auditTrail, id, action, request, auditId,
-                    original.getUser().roles()));
+                    () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, original.getUser().roles())));
         assertEquals(id, e.id());
-        verify(auditTrail).accessDenied(auditId, diffRunAs, action, request, original.getUser().roles());
+        verify(auditTrail).accessDenied(eq(auditId), eq(diffRunAs), eq(action), eq(request), authzInfoRoles(original.getUser().roles()));
 
         // run as different looked up by type
         Authentication runAsDiffType = new Authentication(user, new RealmRef("realm", "file", "node"),
                 new RealmRef(randomAlphaOfLengthBetween(1, 16), randomAlphaOfLengthBetween(5, 12), "node"));
         e = expectThrows(SearchContextMissingException.class,
                 () -> ensureAuthenticatedUserIsSame(runAs, runAsDiffType, auditTrail, id, action, request, auditId,
-                    original.getUser().roles()));
+                    () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, original.getUser().roles())));
         assertEquals(id, e.id());
-        verify(auditTrail).accessDenied(auditId, runAsDiffType, action, request, original.getUser().roles());
+        verify(auditTrail).accessDenied(eq(auditId), eq(runAsDiffType), eq(action), eq(request),
+            authzInfoRoles(original.getUser().roles()));
     }
 
     static class TestScrollSearchContext extends TestSearchContext {

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/IndicesPermissionTests.java

@@ -9,6 +9,7 @@ import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.search.SearchAction;
 import org.elasticsearch.cluster.metadata.AliasMetaData;
+import org.elasticsearch.cluster.metadata.AliasOrIndex;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.Strings;
@@ -35,6 +36,7 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.SortedMap;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -55,14 +57,15 @@ public class IndicesPermissionTests extends ESTestCase {
                 .putAlias(AliasMetaData.builder("_alias"));
         MetaData md = MetaData.builder().put(imbBuilder).build();
         FieldPermissionsCache fieldPermissionsCache = new FieldPermissionsCache(Settings.EMPTY);
+        SortedMap<String, AliasOrIndex> lookup = md.getAliasAndIndexLookup();
 
         // basics:
         Set<BytesReference> query = Collections.singleton(new BytesArray("{}"));
         String[] fields = new String[]{"_field"};
         Role role = Role.builder("_role")
-                .add(new FieldPermissions(fieldPermissionDef(fields, null)), query, IndexPrivilege.ALL, randomBoolean(), "_index")
-                .build();
-        IndicesAccessControl permissions = role.authorize(SearchAction.NAME, Sets.newHashSet("_index"), md, fieldPermissionsCache);
+            .add(new FieldPermissions(fieldPermissionDef(fields, null)), query, IndexPrivilege.ALL, randomBoolean(), "_index")
+            .build();
+        IndicesAccessControl permissions = role.authorize(SearchAction.NAME, Sets.newHashSet("_index"), lookup, fieldPermissionsCache);
         assertThat(permissions.getIndexPermissions("_index"), notNullValue());
         assertTrue(permissions.getIndexPermissions("_index").getFieldPermissions().grantsAccessTo("_field"));
         assertTrue(permissions.getIndexPermissions("_index").getFieldPermissions().hasFieldLevelSecurity());
@@ -72,9 +75,9 @@ public class IndicesPermissionTests extends ESTestCase {
 
         // no document level security:
         role = Role.builder("_role")
-                .add(new FieldPermissions(fieldPermissionDef(fields, null)), null, IndexPrivilege.ALL, randomBoolean(), "_index")
-                .build();
-        permissions = role.authorize(SearchAction.NAME, Sets.newHashSet("_index"), md, fieldPermissionsCache);
+            .add(new FieldPermissions(fieldPermissionDef(fields, null)), null, IndexPrivilege.ALL, randomBoolean(), "_index")
+            .build();
+        permissions = role.authorize(SearchAction.NAME, Sets.newHashSet("_index"), lookup, fieldPermissionsCache);
         assertThat(permissions.getIndexPermissions("_index"), notNullValue());
         assertTrue(permissions.getIndexPermissions("_index").getFieldPermissions().grantsAccessTo("_field"));
         assertTrue(permissions.getIndexPermissions("_index").getFieldPermissions().hasFieldLevelSecurity());
@@ -83,7 +86,7 @@ public class IndicesPermissionTests extends ESTestCase {
 
         // no field level security:
         role = Role.builder("_role").add(new FieldPermissions(), query, IndexPrivilege.ALL, randomBoolean(), "_index").build();
-        permissions = role.authorize(SearchAction.NAME, Sets.newHashSet("_index"), md, fieldPermissionsCache);
+        permissions = role.authorize(SearchAction.NAME, Sets.newHashSet("_index"), lookup, fieldPermissionsCache);
         assertThat(permissions.getIndexPermissions("_index"), notNullValue());
         assertFalse(permissions.getIndexPermissions("_index").getFieldPermissions().hasFieldLevelSecurity());
         assertThat(permissions.getIndexPermissions("_index").getDocumentPermissions().hasDocumentLevelPermissions(), is(true));
@@ -94,7 +97,7 @@ public class IndicesPermissionTests extends ESTestCase {
         role = Role.builder("_role")
                 .add(new FieldPermissions(fieldPermissionDef(fields, null)), query, IndexPrivilege.ALL, randomBoolean(), "_alias")
                 .build();
-        permissions = role.authorize(SearchAction.NAME, Sets.newHashSet("_alias"), md, fieldPermissionsCache);
+        permissions = role.authorize(SearchAction.NAME, Sets.newHashSet("_alias"), lookup, fieldPermissionsCache);
         assertThat(permissions.getIndexPermissions("_index"), notNullValue());
         assertTrue(permissions.getIndexPermissions("_index").getFieldPermissions().grantsAccessTo("_field"));
         assertTrue(permissions.getIndexPermissions("_index").getFieldPermissions().hasFieldLevelSecurity());
@@ -113,9 +116,9 @@ public class IndicesPermissionTests extends ESTestCase {
         String[] allFields = randomFrom(new String[]{"*"}, new String[]{"foo", "*"},
         new String[]{randomAlphaOfLengthBetween(1, 10), "*"});
         role = Role.builder("_role")
-                .add(new FieldPermissions(fieldPermissionDef(allFields, null)), query, IndexPrivilege.ALL, randomBoolean(), "_alias")
-                .build();
-        permissions = role.authorize(SearchAction.NAME, Sets.newHashSet("_alias"), md, fieldPermissionsCache);
+            .add(new FieldPermissions(fieldPermissionDef(allFields, null)), query, IndexPrivilege.ALL, randomBoolean(), "_alias")
+            .build();
+        permissions = role.authorize(SearchAction.NAME, Sets.newHashSet("_alias"), lookup, fieldPermissionsCache);
         assertThat(permissions.getIndexPermissions("_index"), notNullValue());
         assertFalse(permissions.getIndexPermissions("_index").getFieldPermissions().hasFieldLevelSecurity());
         assertThat(permissions.getIndexPermissions("_index").getDocumentPermissions().hasDocumentLevelPermissions(), is(true));
@@ -136,17 +139,17 @@ public class IndicesPermissionTests extends ESTestCase {
                 )
                 .putAlias(AliasMetaData.builder("_alias"));
         md = MetaData.builder(md).put(imbBuilder1).build();
-
+        lookup = md.getAliasAndIndexLookup();
 
         // match all fields with more than one permission
         Set<BytesReference> fooQuery = Collections.singleton(new BytesArray("{foo}"));
         allFields = randomFrom(new String[]{"*"}, new String[]{"foo", "*"},
                 new String[]{randomAlphaOfLengthBetween(1, 10), "*"});
         role = Role.builder("_role")
-                .add(new FieldPermissions(fieldPermissionDef(allFields, null)), fooQuery, IndexPrivilege.ALL, randomBoolean(), "_alias")
-                .add(new FieldPermissions(fieldPermissionDef(allFields, null)), query, IndexPrivilege.ALL, randomBoolean(), "_alias")
-                .build();
-        permissions = role.authorize(SearchAction.NAME, Sets.newHashSet("_alias"), md, fieldPermissionsCache);
+            .add(new FieldPermissions(fieldPermissionDef(allFields, null)), fooQuery, IndexPrivilege.ALL, randomBoolean(), "_alias")
+            .add(new FieldPermissions(fieldPermissionDef(allFields, null)), query, IndexPrivilege.ALL, randomBoolean(), "_alias")
+            .build();
+        permissions = role.authorize(SearchAction.NAME, Sets.newHashSet("_alias"), lookup, fieldPermissionsCache);
         Set<BytesReference> bothQueries = Sets.union(fooQuery, query);
         assertThat(permissions.getIndexPermissions("_index"), notNullValue());
         assertFalse(permissions.getIndexPermissions("_index").getFieldPermissions().hasFieldLevelSecurity());
@@ -178,6 +181,7 @@ public class IndicesPermissionTests extends ESTestCase {
                 .putAlias(AliasMetaData.builder("_alias"));
         MetaData md = MetaData.builder().put(imbBuilder).build();
         FieldPermissionsCache fieldPermissionsCache = new FieldPermissionsCache(Settings.EMPTY);
+        SortedMap<String, AliasOrIndex> lookup = md.getAliasAndIndexLookup();
 
         Set<BytesReference> query = Collections.singleton(new BytesArray("{}"));
         String[] fields = new String[]{"_field"};
@@ -185,7 +189,7 @@ public class IndicesPermissionTests extends ESTestCase {
                 .add(new FieldPermissions(fieldPermissionDef(fields, null)), query, IndexPrivilege.ALL, randomBoolean(), "_index")
                 .add(new FieldPermissions(fieldPermissionDef(null, null)), null, IndexPrivilege.ALL, randomBoolean(), "*")
                 .build();
-        IndicesAccessControl permissions = role.authorize(SearchAction.NAME, Sets.newHashSet("_index"), md, fieldPermissionsCache);
+        IndicesAccessControl permissions = role.authorize(SearchAction.NAME, Sets.newHashSet("_index"), lookup, fieldPermissionsCache);
         assertThat(permissions.getIndexPermissions("_index"), notNullValue());
         assertTrue(permissions.getIndexPermissions("_index").getFieldPermissions().grantsAccessTo("_field"));
         assertFalse(permissions.getIndexPermissions("_index").getFieldPermissions().hasFieldLevelSecurity());
@@ -232,6 +236,7 @@ public class IndicesPermissionTests extends ESTestCase {
                 .put(new IndexMetaData.Builder("a1").settings(indexSettings).numberOfShards(1).numberOfReplicas(0).build(), true)
                 .put(new IndexMetaData.Builder("a2").settings(indexSettings).numberOfShards(1).numberOfReplicas(0).build(), true)
                 .build();
+        SortedMap<String, AliasOrIndex> lookup = metaData.getAliasAndIndexLookup();
 
         FieldPermissionsCache fieldPermissionsCache = new FieldPermissionsCache(Settings.EMPTY);
         IndicesPermission.Group group1 = new IndicesPermission.Group(IndexPrivilege.ALL, new FieldPermissions(), null, randomBoolean(),
@@ -240,7 +245,7 @@ public class IndicesPermissionTests extends ESTestCase {
                 new FieldPermissions(fieldPermissionDef(null, new String[]{"denied_field"})), null, randomBoolean(), "a1");
         IndicesPermission core = new IndicesPermission(group1, group2);
         Map<String, IndicesAccessControl.IndexAccessControl> authzMap =
-                core.authorize(SearchAction.NAME, Sets.newHashSet("a1", "ba"), metaData, fieldPermissionsCache);
+                core.authorize(SearchAction.NAME, Sets.newHashSet("a1", "ba"), lookup, fieldPermissionsCache);
         assertTrue(authzMap.get("a1").getFieldPermissions().grantsAccessTo("denied_field"));
         assertTrue(authzMap.get("a1").getFieldPermissions().grantsAccessTo(randomAlphaOfLength(5)));
         // did not define anything for ba so we allow all
@@ -260,7 +265,7 @@ public class IndicesPermissionTests extends ESTestCase {
                 new FieldPermissions(fieldPermissionDef(new String[] { "*_field2" }, new String[] { "denied_field2" })), null,
                 randomBoolean(), "a2");
         core = new IndicesPermission(group1, group2, group3, group4);
-        authzMap = core.authorize(SearchAction.NAME, Sets.newHashSet("a1", "a2"), metaData, fieldPermissionsCache);
+        authzMap = core.authorize(SearchAction.NAME, Sets.newHashSet("a1", "a2"), lookup, fieldPermissionsCache);
         assertFalse(authzMap.get("a1").getFieldPermissions().hasFieldLevelSecurity());
         assertFalse(authzMap.get("a2").getFieldPermissions().grantsAccessTo("denied_field2"));
         assertFalse(authzMap.get("a2").getFieldPermissions().grantsAccessTo("denied_field"));
@@ -297,11 +302,12 @@ public class IndicesPermissionTests extends ESTestCase {
                         .build(), true)
                 .build();
         FieldPermissionsCache fieldPermissionsCache = new FieldPermissionsCache(Settings.EMPTY);
+        SortedMap<String, AliasOrIndex> lookup = metaData.getAliasAndIndexLookup();
 
         // allow_restricted_indices: false
         IndicesPermission.Group group = new IndicesPermission.Group(IndexPrivilege.ALL, new FieldPermissions(), null, false, "*");
         Map<String, IndicesAccessControl.IndexAccessControl> authzMap = new IndicesPermission(group).authorize(SearchAction.NAME,
-                Sets.newHashSet(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX, RestrictedIndicesNames.SECURITY_INDEX_NAME), metaData,
+                Sets.newHashSet(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX, RestrictedIndicesNames.SECURITY_INDEX_NAME), lookup,
                 fieldPermissionsCache);
         assertThat(authzMap.get(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX).isGranted(), is(false));
         assertThat(authzMap.get(RestrictedIndicesNames.SECURITY_INDEX_NAME).isGranted(), is(false));
@@ -309,7 +315,7 @@ public class IndicesPermissionTests extends ESTestCase {
         // allow_restricted_indices: true
         group = new IndicesPermission.Group(IndexPrivilege.ALL, new FieldPermissions(), null, true, "*");
         authzMap = new IndicesPermission(group).authorize(SearchAction.NAME,
-                Sets.newHashSet(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX, RestrictedIndicesNames.SECURITY_INDEX_NAME), metaData,
+                Sets.newHashSet(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX, RestrictedIndicesNames.SECURITY_INDEX_NAME), lookup,
                 fieldPermissionsCache);
         assertThat(authzMap.get(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX).isGranted(), is(true));
         assertThat(authzMap.get(RestrictedIndicesNames.SECURITY_INDEX_NAME).isGranted(), is(true));

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/interceptor/IndicesAliasesRequestInterceptorTests.java → x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/interceptor/IndicesAliasesRequestInterceptorTests.java

@@ -3,11 +3,13 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.security.action.interceptor;
+package org.elasticsearch.xpack.security.authz.interceptor;
 
 import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesAction;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
+import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
@@ -16,19 +18,25 @@ import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationResult;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.EmptyAuthorizationInfo;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.RequestInfo;
 import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField;
 import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
 import org.elasticsearch.xpack.core.security.authz.permission.DocumentPermissions;
 import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissions;
 import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsDefinition;
-import org.elasticsearch.xpack.core.security.authz.permission.Role;
-import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege;
 import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.security.audit.AuditTrailService;
 
 import java.util.Collections;
+import java.util.Map;
 import java.util.Set;
 
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -58,7 +66,6 @@ public class IndicesAliasesRequestInterceptorTests extends ESTestCase {
         } else {
             queries = null;
         }
-        Role role = Role.builder().add(fieldPermissions, queries, IndexPrivilege.ALL, randomBoolean(), "foo").build();
         final String action = IndicesAliasesAction.NAME;
         IndicesAccessControl accessControl = new IndicesAccessControl(true, Collections.singletonMap("foo",
                 new IndicesAccessControl.IndexAccessControl(true, fieldPermissions,
@@ -76,8 +83,20 @@ public class IndicesAliasesRequestInterceptorTests extends ESTestCase {
         if (randomBoolean()) {
             indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.removeIndex().index("foofoo"));
         }
+        PlainActionFuture<Void> plainActionFuture = new PlainActionFuture<>();
+        RequestInfo requestInfo = new RequestInfo(authentication, indicesAliasesRequest, action);
+        AuthorizationEngine mockEngine = mock(AuthorizationEngine.class);
+        doAnswer(invocationOnMock -> {
+            ActionListener<AuthorizationResult> listener = (ActionListener<AuthorizationResult>) invocationOnMock.getArguments()[3];
+            listener.onResponse(AuthorizationResult.deny());
+            return null;
+        }).when(mockEngine).validateIndexPermissionsAreSubset(eq(requestInfo), eq(EmptyAuthorizationInfo.INSTANCE), any(Map.class),
+            any(ActionListener.class));
         ElasticsearchSecurityException securityException = expectThrows(ElasticsearchSecurityException.class,
-                () -> interceptor.intercept(indicesAliasesRequest, authentication, role, action));
+                () -> {
+                    interceptor.intercept(requestInfo, mockEngine, EmptyAuthorizationInfo.INSTANCE, plainActionFuture);
+                    plainActionFuture.actionGet();
+                });
         assertEquals("Alias requests are not allowed for users who have field or document level security enabled on one of the indices",
                 securityException.getMessage());
     }
@@ -87,15 +106,11 @@ public class IndicesAliasesRequestInterceptorTests extends ESTestCase {
         when(licenseState.copyCurrentLicenseState()).thenReturn(licenseState);
         when(licenseState.isAuthAllowed()).thenReturn(true);
         when(licenseState.isAuditingAllowed()).thenReturn(true);
-        when(licenseState.isDocumentAndFieldLevelSecurityAllowed()).thenReturn(true);
+        when(licenseState.isDocumentAndFieldLevelSecurityAllowed()).thenReturn(randomBoolean());
         ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
         AuditTrailService auditTrailService = new AuditTrailService(Collections.emptyList(), licenseState);
         Authentication authentication = new Authentication(new User("john", "role"), new RealmRef(null, null, null),
                 new RealmRef(null, null, null));
-        Role role = Role.builder()
-                .add(IndexPrivilege.ALL, "alias")
-                .add(IndexPrivilege.READ, "index")
-                .build();
         final String action = IndicesAliasesAction.NAME;
         IndicesAccessControl accessControl = new IndicesAccessControl(true, Collections.emptyMap());
         threadContext.putTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY, accessControl);
@@ -111,10 +126,24 @@ public class IndicesAliasesRequestInterceptorTests extends ESTestCase {
             indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.removeIndex().index("foofoo"));
         }
 
-        ElasticsearchSecurityException securityException = expectThrows(ElasticsearchSecurityException.class,
-                () -> interceptor.intercept(indicesAliasesRequest, authentication, role, action));
-        assertEquals("Adding an alias is not allowed when the alias has more permissions than any of the indices",
+        AuthorizationEngine mockEngine = mock(AuthorizationEngine.class);
+        {
+            PlainActionFuture<Void> plainActionFuture = new PlainActionFuture<>();
+            RequestInfo requestInfo = new RequestInfo(authentication, indicesAliasesRequest, action);
+            doAnswer(invocationOnMock -> {
+                ActionListener<AuthorizationResult> listener = (ActionListener<AuthorizationResult>) invocationOnMock.getArguments()[3];
+                listener.onResponse(AuthorizationResult.deny());
+                return null;
+            }).when(mockEngine).validateIndexPermissionsAreSubset(eq(requestInfo), eq(EmptyAuthorizationInfo.INSTANCE), any(Map.class),
+                any(ActionListener.class));
+            ElasticsearchSecurityException securityException = expectThrows(ElasticsearchSecurityException.class,
+                () -> {
+                    interceptor.intercept(requestInfo, mockEngine, EmptyAuthorizationInfo.INSTANCE, plainActionFuture);
+                    plainActionFuture.actionGet();
+                });
+            assertEquals("Adding an alias is not allowed when the alias has more permissions than any of the indices",
                 securityException.getMessage());
+        }
 
         // swap target and source for success
         final IndicesAliasesRequest successRequest = new IndicesAliasesRequest();
@@ -125,6 +154,18 @@ public class IndicesAliasesRequestInterceptorTests extends ESTestCase {
         if (randomBoolean()) {
             successRequest.addAliasAction(IndicesAliasesRequest.AliasActions.removeIndex().index("foofoo"));
         }
-        interceptor.intercept(successRequest, authentication, role, action);
+
+        {
+            PlainActionFuture<Void> plainActionFuture = new PlainActionFuture<>();
+            RequestInfo requestInfo = new RequestInfo(authentication, successRequest, action);
+            doAnswer(invocationOnMock -> {
+                ActionListener<AuthorizationResult> listener = (ActionListener<AuthorizationResult>) invocationOnMock.getArguments()[3];
+                listener.onResponse(AuthorizationResult.granted());
+                return null;
+            }).when(mockEngine).validateIndexPermissionsAreSubset(eq(requestInfo), eq(EmptyAuthorizationInfo.INSTANCE), any(Map.class),
+                any(ActionListener.class));
+            interceptor.intercept(requestInfo, mockEngine, EmptyAuthorizationInfo.INSTANCE, plainActionFuture);
+            plainActionFuture.actionGet();
+        }
     }
 }

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/interceptor/ResizeRequestInterceptorTests.java → x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/interceptor/ResizeRequestInterceptorTests.java

@@ -3,12 +3,14 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.security.action.interceptor;
+package org.elasticsearch.xpack.security.authz.interceptor;
 
 import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.shrink.ResizeAction;
 import org.elasticsearch.action.admin.indices.shrink.ResizeRequest;
 import org.elasticsearch.action.admin.indices.shrink.ShrinkAction;
+import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
@@ -18,6 +20,10 @@ import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationResult;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.EmptyAuthorizationInfo;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.RequestInfo;
 import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField;
 import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
 import org.elasticsearch.xpack.core.security.authz.permission.DocumentPermissions;
@@ -29,8 +35,12 @@ import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.security.audit.AuditTrailService;
 
 import java.util.Collections;
+import java.util.Map;
 import java.util.Set;
 
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -61,7 +71,6 @@ public class ResizeRequestInterceptorTests extends ESTestCase {
         } else {
             queries = null;
         }
-        Role role = Role.builder().add(fieldPermissions, queries, IndexPrivilege.ALL, randomBoolean(), "foo").build();
         final String action = randomFrom(ShrinkAction.NAME, ResizeAction.NAME);
         IndicesAccessControl accessControl = new IndicesAccessControl(true, Collections.singletonMap("foo",
                 new IndicesAccessControl.IndexAccessControl(true, fieldPermissions,
@@ -71,8 +80,20 @@ public class ResizeRequestInterceptorTests extends ESTestCase {
         ResizeRequestInterceptor resizeRequestInterceptor =
                 new ResizeRequestInterceptor(threadPool, licenseState, auditTrailService);
 
+        PlainActionFuture<Void> plainActionFuture = new PlainActionFuture<>();
+        RequestInfo requestInfo = new RequestInfo(authentication, new ResizeRequest("bar", "foo"), action);
+        AuthorizationEngine mockEngine = mock(AuthorizationEngine.class);
+        doAnswer(invocationOnMock -> {
+            ActionListener<AuthorizationResult> listener = (ActionListener<AuthorizationResult>) invocationOnMock.getArguments()[3];
+            listener.onResponse(AuthorizationResult.deny());
+            return null;
+        }).when(mockEngine).validateIndexPermissionsAreSubset(eq(requestInfo), eq(EmptyAuthorizationInfo.INSTANCE), any(Map.class),
+            any(ActionListener.class));
         ElasticsearchSecurityException securityException = expectThrows(ElasticsearchSecurityException.class,
-                () -> resizeRequestInterceptor.intercept(new ResizeRequest("bar", "foo"), authentication, role, action));
+                () -> {
+                    resizeRequestInterceptor.intercept(requestInfo, mockEngine, EmptyAuthorizationInfo.INSTANCE, plainActionFuture);
+                    plainActionFuture.actionGet();
+                });
         assertEquals("Resize requests are not allowed for users when field or document level security is enabled on the source index",
                 securityException.getMessage());
     }
@@ -97,12 +118,38 @@ public class ResizeRequestInterceptorTests extends ESTestCase {
         threadContext.putTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY, accessControl);
         ResizeRequestInterceptor resizeRequestInterceptor =
                 new ResizeRequestInterceptor(threadPool, licenseState, auditTrailService);
-        ElasticsearchSecurityException securityException = expectThrows(ElasticsearchSecurityException.class,
-                () -> resizeRequestInterceptor.intercept(new ResizeRequest("target", "source"), authentication, role, action));
-        assertEquals("Resizing an index is not allowed when the target index has more permissions than the source index",
+
+        AuthorizationEngine mockEngine = mock(AuthorizationEngine.class);
+        {
+            PlainActionFuture<Void> plainActionFuture = new PlainActionFuture<>();
+            RequestInfo requestInfo = new RequestInfo(authentication, new ResizeRequest("target", "source"), action);
+            doAnswer(invocationOnMock -> {
+                ActionListener<AuthorizationResult> listener = (ActionListener<AuthorizationResult>) invocationOnMock.getArguments()[3];
+                listener.onResponse(AuthorizationResult.deny());
+                return null;
+            }).when(mockEngine).validateIndexPermissionsAreSubset(eq(requestInfo), eq(EmptyAuthorizationInfo.INSTANCE), any(Map.class),
+                any(ActionListener.class));
+            ElasticsearchSecurityException securityException = expectThrows(ElasticsearchSecurityException.class,
+                () -> {
+                    resizeRequestInterceptor.intercept(requestInfo, mockEngine, EmptyAuthorizationInfo.INSTANCE, plainActionFuture);
+                    plainActionFuture.actionGet();
+                });
+            assertEquals("Resizing an index is not allowed when the target index has more permissions than the source index",
                 securityException.getMessage());
+        }
 
         // swap target and source for success
-        resizeRequestInterceptor.intercept(new ResizeRequest("source", "target"), authentication, role, action);
+        {
+            PlainActionFuture<Void> plainActionFuture = new PlainActionFuture<>();
+            RequestInfo requestInfo = new RequestInfo(authentication, new ResizeRequest("source", "target"), action);
+            doAnswer(invocationOnMock -> {
+                ActionListener<AuthorizationResult> listener = (ActionListener<AuthorizationResult>) invocationOnMock.getArguments()[3];
+                listener.onResponse(AuthorizationResult.granted());
+                return null;
+            }).when(mockEngine).validateIndexPermissionsAreSubset(eq(requestInfo), eq(EmptyAuthorizationInfo.INSTANCE), any(Map.class),
+                any(ActionListener.class));
+            resizeRequestInterceptor.intercept(requestInfo, mockEngine, EmptyAuthorizationInfo.INSTANCE, plainActionFuture);
+            plainActionFuture.actionGet();
+        }
     }
 }

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java

@@ -12,6 +12,7 @@ import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
 import org.elasticsearch.action.get.GetAction;
 import org.elasticsearch.action.index.IndexAction;
 import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.client.security.user.privileges.Role.ClusterPrivilegeName;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
@@ -27,9 +28,13 @@ import org.elasticsearch.license.TestUtils.UpdatableLicenseState;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.transport.TransportRequest;
+import org.elasticsearch.transport.TransportRequest.Empty;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.security.action.saml.SamlAuthenticateAction;
 import org.elasticsearch.xpack.core.security.action.user.PutUserAction;
+import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authc.Authentication.AuthenticationType;
+import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.IndicesPrivileges;
 import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
@@ -42,6 +47,13 @@ import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterP
 import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege;
 import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
 import org.elasticsearch.xpack.core.security.authz.store.RoleRetrievalResult;
+import org.elasticsearch.xpack.core.security.user.AnonymousUser;
+import org.elasticsearch.xpack.core.security.user.SystemUser;
+import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.xpack.core.security.user.XPackUser;
+import org.elasticsearch.xpack.security.audit.AuditUtil;
+import org.elasticsearch.xpack.security.authc.ApiKeyService;
+import org.elasticsearch.xpack.security.authc.ApiKeyService.ApiKeyRoleDescriptors;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 
 import java.io.IOException;
@@ -62,6 +74,8 @@ import static org.elasticsearch.mock.orig.Mockito.times;
 import static org.elasticsearch.mock.orig.Mockito.verifyNoMoreInteractions;
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.is;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.anySetOf;
 import static org.mockito.Matchers.eq;
@@ -128,7 +142,7 @@ public class CompositeRolesStoreTests extends ESTestCase {
         when(fileRolesStore.roleDescriptors(Collections.singleton("no_fls_dls"))).thenReturn(Collections.singleton(noFlsDlsRole));
         CompositeRolesStore compositeRolesStore = new CompositeRolesStore(Settings.EMPTY, fileRolesStore, nativeRolesStore,
                 reservedRolesStore, mock(NativePrivilegeStore.class), Collections.emptyList(),
-                new ThreadContext(Settings.EMPTY), licenseState, cache);
+                new ThreadContext(Settings.EMPTY), licenseState, cache, mock(ApiKeyService.class));
 
         PlainActionFuture<Role> roleFuture = new PlainActionFuture<>();
         compositeRolesStore.roles(Collections.singleton("fls"), roleFuture);
@@ -193,7 +207,7 @@ public class CompositeRolesStoreTests extends ESTestCase {
         when(fileRolesStore.roleDescriptors(Collections.singleton("no_fls_dls"))).thenReturn(Collections.singleton(noFlsDlsRole));
         CompositeRolesStore compositeRolesStore = new CompositeRolesStore(Settings.EMPTY, fileRolesStore, nativeRolesStore,
                 reservedRolesStore, mock(NativePrivilegeStore.class), Collections.emptyList(),
-                new ThreadContext(Settings.EMPTY), licenseState, cache);
+                new ThreadContext(Settings.EMPTY), licenseState, cache, mock(ApiKeyService.class));
 
         PlainActionFuture<Role> roleFuture = new PlainActionFuture<>();
         compositeRolesStore.roles(Collections.singleton("fls"), roleFuture);
@@ -235,7 +249,7 @@ public class CompositeRolesStoreTests extends ESTestCase {
         final CompositeRolesStore compositeRolesStore =
                 new CompositeRolesStore(SECURITY_ENABLED_SETTINGS, fileRolesStore, nativeRolesStore, reservedRolesStore,
                         nativePrivilegeStore, Collections.emptyList(), new ThreadContext(SECURITY_ENABLED_SETTINGS),
-                        new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache);
+                        new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, mock(ApiKeyService.class));
         verify(fileRolesStore).addListener(any(Consumer.class)); // adds a listener in ctor
 
         final String roleName = randomAlphaOfLengthBetween(1, 10);
@@ -286,7 +300,7 @@ public class CompositeRolesStoreTests extends ESTestCase {
             .build();
         final CompositeRolesStore compositeRolesStore = new CompositeRolesStore(settings, fileRolesStore, nativeRolesStore,
             reservedRolesStore, mock(NativePrivilegeStore.class), Collections.emptyList(), new ThreadContext(settings),
-            new XPackLicenseState(settings), cache);
+            new XPackLicenseState(settings), cache, mock(ApiKeyService.class));
         verify(fileRolesStore).addListener(any(Consumer.class)); // adds a listener in ctor
 
         final String roleName = randomAlphaOfLengthBetween(1, 10);
@@ -320,7 +334,7 @@ public class CompositeRolesStoreTests extends ESTestCase {
         final CompositeRolesStore compositeRolesStore =
             new CompositeRolesStore(SECURITY_ENABLED_SETTINGS, fileRolesStore, nativeRolesStore, reservedRolesStore,
                 mock(NativePrivilegeStore.class), Collections.emptyList(), new ThreadContext(SECURITY_ENABLED_SETTINGS),
-                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache);
+                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, mock(ApiKeyService.class));
         verify(fileRolesStore).addListener(any(Consumer.class)); // adds a listener in ctor
 
         final String roleName = randomAlphaOfLengthBetween(1, 10);
@@ -397,7 +411,8 @@ public class CompositeRolesStoreTests extends ESTestCase {
         final CompositeRolesStore compositeRolesStore =
                 new CompositeRolesStore(SECURITY_ENABLED_SETTINGS, fileRolesStore, nativeRolesStore, reservedRolesStore,
                                 mock(NativePrivilegeStore.class), Arrays.asList(inMemoryProvider1, inMemoryProvider2),
-                                new ThreadContext(SECURITY_ENABLED_SETTINGS), new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache);
+                                new ThreadContext(SECURITY_ENABLED_SETTINGS), new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache,
+                    mock(ApiKeyService.class));
 
         final Set<String> roleNames = Sets.newHashSet("roleA", "roleB", "unknown");
         PlainActionFuture<Role> future = new PlainActionFuture<>();
@@ -458,8 +473,8 @@ public class CompositeRolesStoreTests extends ESTestCase {
                         .settings(Settings.builder().put("index.version.created", Version.CURRENT).build())
                         .numberOfShards(1).numberOfReplicas(0).build(), true)
                 .build();
-        Map<String, IndicesAccessControl.IndexAccessControl> acls =
-                role.indices().authorize("indices:data/read/search", Collections.singleton("test"), metaData, cache);
+        Map<String, IndicesAccessControl.IndexAccessControl> acls = role.indices().authorize("indices:data/read/search",
+            Collections.singleton("test"), metaData.getAliasAndIndexLookup(), cache);
         assertFalse(acls.isEmpty());
         assertTrue(acls.get("test").getFieldPermissions().grantsAccessTo("L1.foo"));
         assertFalse(acls.get("test").getFieldPermissions().grantsAccessTo("L2.foo"));
@@ -600,8 +615,9 @@ public class CompositeRolesStoreTests extends ESTestCase {
 
         final CompositeRolesStore compositeRolesStore =
             new CompositeRolesStore(SECURITY_ENABLED_SETTINGS, fileRolesStore, nativeRolesStore, reservedRolesStore,
-                                    mock(NativePrivilegeStore.class), Arrays.asList(inMemoryProvider1, failingProvider),
-                                    new ThreadContext(SECURITY_ENABLED_SETTINGS), new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache);
+                mock(NativePrivilegeStore.class), Arrays.asList(inMemoryProvider1, failingProvider),
+                new ThreadContext(SECURITY_ENABLED_SETTINGS), new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache,
+                mock(ApiKeyService.class));
 
         final Set<String> roleNames = Sets.newHashSet("roleA", "roleB", "unknown");
         PlainActionFuture<Role> future = new PlainActionFuture<>();
@@ -643,7 +659,7 @@ public class CompositeRolesStoreTests extends ESTestCase {
         xPackLicenseState.update(randomFrom(OperationMode.BASIC, OperationMode.GOLD, OperationMode.STANDARD), true, null);
         CompositeRolesStore compositeRolesStore = new CompositeRolesStore(
             Settings.EMPTY, fileRolesStore, nativeRolesStore, reservedRolesStore, mock(NativePrivilegeStore.class),
-            Arrays.asList(inMemoryProvider), new ThreadContext(Settings.EMPTY), xPackLicenseState, cache);
+            Arrays.asList(inMemoryProvider), new ThreadContext(Settings.EMPTY), xPackLicenseState, cache, mock(ApiKeyService.class));
 
         Set<String> roleNames = Sets.newHashSet("roleA");
         PlainActionFuture<Role> future = new PlainActionFuture<>();
@@ -655,7 +671,7 @@ public class CompositeRolesStoreTests extends ESTestCase {
 
         compositeRolesStore = new CompositeRolesStore(
             Settings.EMPTY, fileRolesStore, nativeRolesStore, reservedRolesStore, mock(NativePrivilegeStore.class),
-            Arrays.asList(inMemoryProvider), new ThreadContext(Settings.EMPTY), xPackLicenseState, cache);
+            Arrays.asList(inMemoryProvider), new ThreadContext(Settings.EMPTY), xPackLicenseState, cache, mock(ApiKeyService.class));
         // these licenses allow custom role providers
         xPackLicenseState.update(randomFrom(OperationMode.PLATINUM, OperationMode.TRIAL), true, null);
         roleNames = Sets.newHashSet("roleA");
@@ -669,7 +685,7 @@ public class CompositeRolesStoreTests extends ESTestCase {
         // license expired, don't allow custom role providers
         compositeRolesStore = new CompositeRolesStore(
             Settings.EMPTY, fileRolesStore, nativeRolesStore, reservedRolesStore, mock(NativePrivilegeStore.class),
-            Arrays.asList(inMemoryProvider), new ThreadContext(Settings.EMPTY), xPackLicenseState, cache);
+            Arrays.asList(inMemoryProvider), new ThreadContext(Settings.EMPTY), xPackLicenseState, cache, mock(ApiKeyService.class));
         xPackLicenseState.update(randomFrom(OperationMode.PLATINUM, OperationMode.TRIAL), false, null);
         roleNames = Sets.newHashSet("roleA");
         future = new PlainActionFuture<>();
@@ -694,7 +710,7 @@ public class CompositeRolesStoreTests extends ESTestCase {
         CompositeRolesStore compositeRolesStore = new CompositeRolesStore(
                 Settings.EMPTY, fileRolesStore, nativeRolesStore, reservedRolesStore,
                 mock(NativePrivilegeStore.class), Collections.emptyList(), new ThreadContext(Settings.EMPTY),
-                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache) {
+                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, mock(ApiKeyService.class)) {
             @Override
             public void invalidateAll() {
                 numInvalidation.incrementAndGet();
@@ -746,7 +762,7 @@ public class CompositeRolesStoreTests extends ESTestCase {
         CompositeRolesStore compositeRolesStore = new CompositeRolesStore(SECURITY_ENABLED_SETTINGS,
                 fileRolesStore, nativeRolesStore, reservedRolesStore,
                 mock(NativePrivilegeStore.class), Collections.emptyList(), new ThreadContext(SECURITY_ENABLED_SETTINGS),
-                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache) {
+                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, mock(ApiKeyService.class)) {
             @Override
             public void invalidateAll() {
                 numInvalidation.incrementAndGet();
@@ -764,6 +780,209 @@ public class CompositeRolesStoreTests extends ESTestCase {
         assertEquals(2, numInvalidation.get());
     }
 
+    public void testDefaultRoleUserWithoutRoles() {
+        final FileRolesStore fileRolesStore = mock(FileRolesStore.class);
+        doCallRealMethod().when(fileRolesStore).accept(any(Set.class), any(ActionListener.class));
+        final NativeRolesStore nativeRolesStore = mock(NativeRolesStore.class);
+        doCallRealMethod().when(nativeRolesStore).accept(any(Set.class), any(ActionListener.class));
+        when(fileRolesStore.roleDescriptors(anySetOf(String.class))).thenReturn(Collections.emptySet());
+        doAnswer((invocationOnMock) -> {
+            ActionListener<RoleRetrievalResult> callback = (ActionListener<RoleRetrievalResult>) invocationOnMock.getArguments()[1];
+            callback.onResponse(RoleRetrievalResult.failure(new RuntimeException("intentionally failed!")));
+            return null;
+        }).when(nativeRolesStore).getRoleDescriptors(isA(Set.class), any(ActionListener.class));
+        final ReservedRolesStore reservedRolesStore = spy(new ReservedRolesStore());
+
+        final CompositeRolesStore compositeRolesStore =
+            new CompositeRolesStore(SECURITY_ENABLED_SETTINGS, fileRolesStore, nativeRolesStore, reservedRolesStore,
+                mock(NativePrivilegeStore.class), Collections.emptyList(), new ThreadContext(SECURITY_ENABLED_SETTINGS),
+                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, mock(ApiKeyService.class));
+        verify(fileRolesStore).addListener(any(Consumer.class)); // adds a listener in ctor
+
+
+        PlainActionFuture<Role> rolesFuture = new PlainActionFuture<>();
+        final User user = new User("no role user");
+        Authentication auth = new Authentication(user, new RealmRef("name", "type", "node"), null);
+        compositeRolesStore.getRoles(user, auth, rolesFuture);
+        final Role roles = rolesFuture.actionGet();
+        assertEquals(Role.EMPTY, roles);
+    }
+
+    public void testAnonymousUserEnabledRoleAdded() {
+        Settings settings = Settings.builder()
+            .put(SECURITY_ENABLED_SETTINGS)
+            .put(AnonymousUser.ROLES_SETTING.getKey(), "anonymous_user_role")
+            .build();
+        final FileRolesStore fileRolesStore = mock(FileRolesStore.class);
+        doCallRealMethod().when(fileRolesStore).accept(any(Set.class), any(ActionListener.class));
+        final NativeRolesStore nativeRolesStore = mock(NativeRolesStore.class);
+        doCallRealMethod().when(nativeRolesStore).accept(any(Set.class), any(ActionListener.class));
+        doAnswer(invocationOnMock -> {
+            Set<String> names = (Set<String>) invocationOnMock.getArguments()[0];
+            if (names.size() == 1 && names.contains("anonymous_user_role")) {
+                RoleDescriptor rd = new RoleDescriptor("anonymous_user_role", null, null, null);
+                return Collections.singleton(rd);
+            }
+            return Collections.emptySet();
+        }).
+        when(fileRolesStore).roleDescriptors(anySetOf(String.class));
+        doAnswer((invocationOnMock) -> {
+            ActionListener<RoleRetrievalResult> callback = (ActionListener<RoleRetrievalResult>) invocationOnMock.getArguments()[1];
+            callback.onResponse(RoleRetrievalResult.failure(new RuntimeException("intentionally failed!")));
+            return null;
+        }).when(nativeRolesStore).getRoleDescriptors(isA(Set.class), any(ActionListener.class));
+        final ReservedRolesStore reservedRolesStore = spy(new ReservedRolesStore());
+
+        final CompositeRolesStore compositeRolesStore =
+            new CompositeRolesStore(settings, fileRolesStore, nativeRolesStore, reservedRolesStore,
+                mock(NativePrivilegeStore.class), Collections.emptyList(), new ThreadContext(settings),
+                new XPackLicenseState(settings), cache, mock(ApiKeyService.class));
+        verify(fileRolesStore).addListener(any(Consumer.class)); // adds a listener in ctor
+
+        PlainActionFuture<Role> rolesFuture = new PlainActionFuture<>();
+        final User user = new User("no role user");
+        Authentication auth = new Authentication(user, new RealmRef("name", "type", "node"), null);
+        compositeRolesStore.getRoles(user, auth, rolesFuture);
+        final Role roles = rolesFuture.actionGet();
+        assertThat(Arrays.asList(roles.names()), hasItem("anonymous_user_role"));
+    }
+
+    public void testDoesNotUseRolesStoreForXPackUser() {
+        final FileRolesStore fileRolesStore = mock(FileRolesStore.class);
+        doCallRealMethod().when(fileRolesStore).accept(any(Set.class), any(ActionListener.class));
+        final NativeRolesStore nativeRolesStore = mock(NativeRolesStore.class);
+        doCallRealMethod().when(nativeRolesStore).accept(any(Set.class), any(ActionListener.class));
+        when(fileRolesStore.roleDescriptors(anySetOf(String.class))).thenReturn(Collections.emptySet());
+        doAnswer((invocationOnMock) -> {
+            ActionListener<RoleRetrievalResult> callback = (ActionListener<RoleRetrievalResult>) invocationOnMock.getArguments()[1];
+            callback.onResponse(RoleRetrievalResult.failure(new RuntimeException("intentionally failed!")));
+            return null;
+        }).when(nativeRolesStore).getRoleDescriptors(isA(Set.class), any(ActionListener.class));
+        final ReservedRolesStore reservedRolesStore = spy(new ReservedRolesStore());
+
+        final CompositeRolesStore compositeRolesStore =
+            new CompositeRolesStore(SECURITY_ENABLED_SETTINGS, fileRolesStore, nativeRolesStore, reservedRolesStore,
+                mock(NativePrivilegeStore.class), Collections.emptyList(), new ThreadContext(SECURITY_ENABLED_SETTINGS),
+                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, mock(ApiKeyService.class));
+        verify(fileRolesStore).addListener(any(Consumer.class)); // adds a listener in ctor
+
+        PlainActionFuture<Role> rolesFuture = new PlainActionFuture<>();
+        Authentication auth = new Authentication(XPackUser.INSTANCE, new RealmRef("name", "type", "node"), null);
+        compositeRolesStore.getRoles(XPackUser.INSTANCE, auth, rolesFuture);
+        final Role roles = rolesFuture.actionGet();
+        assertThat(roles, equalTo(XPackUser.ROLE));
+        verifyNoMoreInteractions(fileRolesStore, nativeRolesStore, reservedRolesStore);
+    }
+
+    public void testGetRolesForSystemUserThrowsException() {
+        final FileRolesStore fileRolesStore = mock(FileRolesStore.class);
+        doCallRealMethod().when(fileRolesStore).accept(any(Set.class), any(ActionListener.class));
+        final NativeRolesStore nativeRolesStore = mock(NativeRolesStore.class);
+        doCallRealMethod().when(nativeRolesStore).accept(any(Set.class), any(ActionListener.class));
+        when(fileRolesStore.roleDescriptors(anySetOf(String.class))).thenReturn(Collections.emptySet());
+        doAnswer((invocationOnMock) -> {
+            ActionListener<RoleRetrievalResult> callback = (ActionListener<RoleRetrievalResult>) invocationOnMock.getArguments()[1];
+            callback.onResponse(RoleRetrievalResult.failure(new RuntimeException("intentionally failed!")));
+            return null;
+        }).when(nativeRolesStore).getRoleDescriptors(isA(Set.class), any(ActionListener.class));
+        final ReservedRolesStore reservedRolesStore = spy(new ReservedRolesStore());
+
+        final CompositeRolesStore compositeRolesStore =
+            new CompositeRolesStore(SECURITY_ENABLED_SETTINGS, fileRolesStore, nativeRolesStore, reservedRolesStore,
+                mock(NativePrivilegeStore.class), Collections.emptyList(), new ThreadContext(SECURITY_ENABLED_SETTINGS),
+                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, mock(ApiKeyService.class));
+        verify(fileRolesStore).addListener(any(Consumer.class)); // adds a listener in ctor
+        IllegalArgumentException iae = expectThrows(IllegalArgumentException.class,
+            () -> compositeRolesStore.getRoles(SystemUser.INSTANCE, null, null));
+        assertEquals("the user [_system] is the system user and we should never try to get its roles", iae.getMessage());
+    }
+
+    public void testApiKeyAuthUsesApiKeyService() throws IOException {
+        final FileRolesStore fileRolesStore = mock(FileRolesStore.class);
+        doCallRealMethod().when(fileRolesStore).accept(any(Set.class), any(ActionListener.class));
+        final NativeRolesStore nativeRolesStore = mock(NativeRolesStore.class);
+        doCallRealMethod().when(nativeRolesStore).accept(any(Set.class), any(ActionListener.class));
+        when(fileRolesStore.roleDescriptors(anySetOf(String.class))).thenReturn(Collections.emptySet());
+        doAnswer((invocationOnMock) -> {
+            ActionListener<RoleRetrievalResult> callback = (ActionListener<RoleRetrievalResult>) invocationOnMock.getArguments()[1];
+            callback.onResponse(RoleRetrievalResult.failure(new RuntimeException("intentionally failed!")));
+            return null;
+        }).when(nativeRolesStore).getRoleDescriptors(isA(Set.class), any(ActionListener.class));
+        final ReservedRolesStore reservedRolesStore = spy(new ReservedRolesStore());
+        ThreadContext threadContext = new ThreadContext(SECURITY_ENABLED_SETTINGS);
+        ApiKeyService apiKeyService = mock(ApiKeyService.class);
+        NativePrivilegeStore nativePrivStore = mock(NativePrivilegeStore.class);
+        doAnswer(invocationOnMock -> {
+            ActionListener<Collection<ApplicationPrivilegeDescriptor>> listener =
+                (ActionListener<Collection<ApplicationPrivilegeDescriptor>>) invocationOnMock.getArguments()[2];
+            listener.onResponse(Collections.emptyList());
+            return Void.TYPE;
+        }).when(nativePrivStore).getPrivileges(any(Collection.class), any(Collection.class), any(ActionListener.class));
+        final CompositeRolesStore compositeRolesStore =
+            new CompositeRolesStore(SECURITY_ENABLED_SETTINGS, fileRolesStore, nativeRolesStore, reservedRolesStore,
+                nativePrivStore, Collections.emptyList(), new ThreadContext(SECURITY_ENABLED_SETTINGS),
+                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, apiKeyService);
+        AuditUtil.getOrGenerateRequestId(threadContext);
+        final Authentication authentication = new Authentication(new User("test api key user", "superuser"),
+            new RealmRef("_es_api_key", "_es_api_key", "node"), null, Version.CURRENT, AuthenticationType.API_KEY, Collections.emptyMap());
+        doAnswer(invocationOnMock -> {
+            ActionListener<ApiKeyRoleDescriptors> listener = (ActionListener<ApiKeyRoleDescriptors>) invocationOnMock.getArguments()[1];
+            listener.onResponse(new ApiKeyRoleDescriptors("keyId",
+                Collections.singletonList(ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR), null));
+            return Void.TYPE;
+        }).when(apiKeyService).getRoleForApiKey(eq(authentication), any(ActionListener.class));
+
+        PlainActionFuture<Role> roleFuture = new PlainActionFuture<>();
+        compositeRolesStore.getRoles(authentication.getUser(), authentication, roleFuture);
+        roleFuture.actionGet();
+
+        verify(apiKeyService).getRoleForApiKey(eq(authentication), any(ActionListener.class));
+    }
+
+    public void testApiKeyAuthUsesApiKeyServiceWithScopedRole() throws IOException {
+        final FileRolesStore fileRolesStore = mock(FileRolesStore.class);
+        doCallRealMethod().when(fileRolesStore).accept(any(Set.class), any(ActionListener.class));
+        final NativeRolesStore nativeRolesStore = mock(NativeRolesStore.class);
+        doCallRealMethod().when(nativeRolesStore).accept(any(Set.class), any(ActionListener.class));
+        when(fileRolesStore.roleDescriptors(anySetOf(String.class))).thenReturn(Collections.emptySet());
+        doAnswer((invocationOnMock) -> {
+            ActionListener<RoleRetrievalResult> callback = (ActionListener<RoleRetrievalResult>) invocationOnMock.getArguments()[1];
+            callback.onResponse(RoleRetrievalResult.failure(new RuntimeException("intentionally failed!")));
+            return null;
+        }).when(nativeRolesStore).getRoleDescriptors(isA(Set.class), any(ActionListener.class));
+        final ReservedRolesStore reservedRolesStore = spy(new ReservedRolesStore());
+        ThreadContext threadContext = new ThreadContext(SECURITY_ENABLED_SETTINGS);
+        ApiKeyService apiKeyService = mock(ApiKeyService.class);
+        NativePrivilegeStore nativePrivStore = mock(NativePrivilegeStore.class);
+        doAnswer(invocationOnMock -> {
+            ActionListener<Collection<ApplicationPrivilegeDescriptor>> listener =
+                (ActionListener<Collection<ApplicationPrivilegeDescriptor>>) invocationOnMock.getArguments()[2];
+            listener.onResponse(Collections.emptyList());
+            return Void.TYPE;
+        }).when(nativePrivStore).getPrivileges(any(Collection.class), any(Collection.class), any(ActionListener.class));
+        final CompositeRolesStore compositeRolesStore =
+            new CompositeRolesStore(SECURITY_ENABLED_SETTINGS, fileRolesStore, nativeRolesStore, reservedRolesStore,
+                nativePrivStore, Collections.emptyList(), new ThreadContext(SECURITY_ENABLED_SETTINGS),
+                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, apiKeyService);
+        AuditUtil.getOrGenerateRequestId(threadContext);
+        final Authentication authentication = new Authentication(new User("test api key user", "api_key"),
+            new RealmRef("_es_api_key", "_es_api_key", "node"), null, Version.CURRENT, AuthenticationType.API_KEY, Collections.emptyMap());
+        doAnswer(invocationOnMock -> {
+            ActionListener<ApiKeyRoleDescriptors> listener = (ActionListener<ApiKeyRoleDescriptors>) invocationOnMock.getArguments()[1];
+            listener.onResponse(new ApiKeyRoleDescriptors("keyId",
+                Collections.singletonList(new RoleDescriptor("a-role", new String[] { ClusterPrivilegeName.ALL }, null, null)),
+                Collections.singletonList(
+                    new RoleDescriptor("scoped-role", new String[] { ClusterPrivilegeName.MANAGE_SECURITY }, null, null))));
+            return Void.TYPE;
+        }).when(apiKeyService).getRoleForApiKey(eq(authentication), any(ActionListener.class));
+
+        PlainActionFuture<Role> roleFuture = new PlainActionFuture<>();
+        compositeRolesStore.getRoles(authentication.getUser(), authentication, roleFuture);
+        Role role = roleFuture.actionGet();
+        assertThat(role.checkClusterAction("cluster:admin/foo", Empty.INSTANCE), is(false));
+        verify(apiKeyService).getRoleForApiKey(eq(authentication), any(ActionListener.class));
+    }
+
     private static class InMemoryRolesProvider implements BiConsumer<Set<String>, ActionListener<RoleRetrievalResult>> {
         private final Function<Set<String>, RoleRetrievalResult> roleDescriptorsFunc;
 

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterTests.java

@@ -25,8 +25,6 @@ import org.elasticsearch.transport.TransportSettings;
 import org.elasticsearch.xpack.core.security.SecurityContext;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef;
-import org.elasticsearch.xpack.core.security.authz.permission.Role;
-import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
 import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.core.security.user.XPackUser;
@@ -37,7 +35,6 @@ import org.junit.Before;
 import java.io.IOException;
 import java.util.Collections;
 
-import static org.elasticsearch.mock.orig.Mockito.times;
 import static org.elasticsearch.xpack.core.security.support.Exceptions.authenticationError;
 import static org.elasticsearch.xpack.core.security.support.Exceptions.authorizationError;
 import static org.hamcrest.Matchers.equalTo;
@@ -89,7 +86,7 @@ public class ServerTransportFilterTests extends ESTestCase {
         PlainActionFuture<Void> future = new PlainActionFuture<>();
         filter.inbound("_action", request, channel, future);
         //future.get(); // don't block it's not called really just mocked
-        verify(authzService).authorize(authentication, "_action", request, null, null);
+        verify(authzService).authorize(eq(authentication), eq("_action"), eq(request), any(ActionListener.class));
     }
 
     public void testInboundDestructiveOperations() throws Exception {
@@ -113,7 +110,7 @@ public class ServerTransportFilterTests extends ESTestCase {
             verify(listener).onFailure(isA(IllegalArgumentException.class));
             verifyNoMoreInteractions(authzService);
         } else {
-            verify(authzService).authorize(authentication, action, request, null, null);
+            verify(authzService).authorize(eq(authentication), eq(action), eq(request), any(ActionListener.class));
         }
     }
 
@@ -148,18 +145,11 @@ public class ServerTransportFilterTests extends ESTestCase {
             callback.onResponse(authentication);
             return Void.TYPE;
         }).when(authcService).authenticate(eq("_action"), eq(request), eq((User)null), any(ActionListener.class));
-        final Role empty = Role.EMPTY;
-        doAnswer((i) -> {
-            ActionListener callback =
-                    (ActionListener) i.getArguments()[2];
-            callback.onResponse(empty);
-            return Void.TYPE;
-        }).when(authzService).roles(any(User.class), any(Authentication.class), any(ActionListener.class));
         when(authentication.getVersion()).thenReturn(Version.CURRENT);
         when(authentication.getUser()).thenReturn(XPackUser.INSTANCE);
         PlainActionFuture<Void> future = new PlainActionFuture<>();
-        doThrow(authorizationError("authz failed")).when(authzService).authorize(authentication, "_action", request,
-                empty, null);
+        doThrow(authorizationError("authz failed"))
+            .when(authzService).authorize(eq(authentication), eq("_action"), eq(request), any(ActionListener.class));
         ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class, () -> {
             filter.inbound("_action", request, channel, future);
             future.actionGet();
@@ -186,12 +176,6 @@ public class ServerTransportFilterTests extends ESTestCase {
         ServerTransportFilter filter = getNodeFilter(true);
         TransportRequest request = mock(TransportRequest.class);
         Authentication authentication = new Authentication(new User("test", "superuser"), new RealmRef("test", "test", "node1"), null);
-        doAnswer((i) -> {
-            ActionListener callback =
-                    (ActionListener) i.getArguments()[2];
-            callback.onResponse(authentication.getUser().equals(i.getArguments()[0]) ? ReservedRolesStore.SUPERUSER_ROLE : null);
-            return Void.TYPE;
-        }).when(authzService).roles(any(User.class), any(Authentication.class), any(ActionListener.class));
         doAnswer((i) -> {
             ActionListener callback =
                     (ActionListener) i.getArguments()[3];
@@ -207,13 +191,11 @@ public class ServerTransportFilterTests extends ESTestCase {
 
         filter.inbound(internalAction, request, channel, new PlainActionFuture<>());
         verify(authcService).authenticate(eq(internalAction), eq(request), eq((User)null), any(ActionListener.class));
-        verify(authzService).roles(eq(authentication.getUser()), any(Authentication.class), any(ActionListener.class));
-        verify(authzService).authorize(authentication, internalAction, request, ReservedRolesStore.SUPERUSER_ROLE, null);
+        verify(authzService).authorize(eq(authentication), eq(internalAction), eq(request), any(ActionListener.class));
 
         filter.inbound(nodeOrShardAction, request, channel, new PlainActionFuture<>());
         verify(authcService).authenticate(eq(nodeOrShardAction), eq(request), eq((User)null), any(ActionListener.class));
-        verify(authzService, times(2)).roles(eq(authentication.getUser()), any(Authentication.class), any(ActionListener.class));
-        verify(authzService).authorize(authentication, nodeOrShardAction, request, ReservedRolesStore.SUPERUSER_ROLE, null);
+        verify(authzService).authorize(eq(authentication), eq(nodeOrShardAction), eq(request), any(ActionListener.class));
         verifyNoMoreInteractions(authcService, authzService);
     }
 

x-pack/qa/security-example-spi-extension/build.gradle

@@ -2,7 +2,7 @@ apply plugin: 'elasticsearch.esplugin'
 
 esplugin {
   name 'spi-extension'
-  description 'An example spi extension pluing for xpack security'
+  description 'An example spi extension plugin for security'
   classname 'org.elasticsearch.example.SpiExtensionPlugin'
   extendedPlugins = ['x-pack-security']
 }