Deprecate permission over aliases (#38059)

This PR generates deprecation log entries for each Role Descriptor,
used for building a Role, when the Role Descriptor grants more privileges
for an alias compared to an index that the alias points to. This is done in
preparation for the removal of the ability to define privileges over aliases.
There is one log entry for each "role descriptor name"-"alias name" pair.
On such a notice, the administrator is expected to modify the Role Descriptor
definition so that the name pattern for index names does not cover aliases.

Caveats:
* Role Descriptors that are not used in any authorization process,
either because they are not mapped to any user or the user they are mapped to
is not used by clients, are not be checked.
* Role Descriptors are merged when building the effective Role that is used in
the authorization process. Therefore some Role Descriptors can overlap others,
so even if one matches aliases in a deprecated way, and it is reported as such,
it is not at risk from the breaking behavior in the current role mapping configuration
and index-alias configuration. It is still reported because it is a best practice to
change its definition, or remove offending aliases.

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java

@@ -53,7 +53,7 @@ public final class IndicesPermission {
         this.groups = groups;
     }
 
-    static Predicate<String> indexMatcher(Collection<String> indices) {
+    public static Predicate<String> indexMatcher(Collection<String> indices) {
         Set<String> exactMatch = new HashSet<>();
         List<String> nonExactMatch = new ArrayList<>();
         for (String indexPattern : indices) {

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java

@@ -188,6 +188,7 @@ import org.elasticsearch.xpack.security.authz.interceptor.ResizeRequestIntercept
 import org.elasticsearch.xpack.security.authz.interceptor.SearchRequestInterceptor;
 import org.elasticsearch.xpack.security.authz.interceptor.UpdateRequestInterceptor;
 import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore;
+import org.elasticsearch.xpack.security.authz.store.DeprecationRoleDescriptorConsumer;
 import org.elasticsearch.xpack.security.authz.store.FileRolesStore;
 import org.elasticsearch.xpack.security.authz.store.NativePrivilegeStore;
 import org.elasticsearch.xpack.security.authz.store.NativeRolesStore;
@@ -452,8 +453,10 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
             threadPool);
         components.add(apiKeyService);
         final CompositeRolesStore allRolesStore = new CompositeRolesStore(settings, fileRolesStore, nativeRolesStore, reservedRolesStore,
-            privilegeStore, rolesProviders, threadPool.getThreadContext(), getLicenseState(), fieldPermissionsCache, apiKeyService);
+            privilegeStore, rolesProviders, threadPool.getThreadContext(), getLicenseState(), fieldPermissionsCache, apiKeyService,
+            new DeprecationRoleDescriptorConsumer(clusterService, threadPool));
         securityIndex.get().addIndexStateListener(allRolesStore::onSecurityIndexStateChange);
+
         // to keep things simple, just invalidate all cached entries on license change. this happens so rarely that the impact should be
         // minimal
         getLicenseState().addListener(allRolesStore::invalidateAll);

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java

@@ -62,6 +62,7 @@ import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.function.BiConsumer;
+import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
@@ -100,6 +101,7 @@ public class CompositeRolesStore {
     private final NativeRolesStore nativeRolesStore;
     private final NativePrivilegeStore privilegeStore;
     private final XPackLicenseState licenseState;
+    private final Consumer<Collection<RoleDescriptor>> effectiveRoleDescriptorsConsumer;
     private final FieldPermissionsCache fieldPermissionsCache;
     private final Cache<RoleKey, Role> roleCache;
     private final Cache<String, Boolean> negativeLookupCache;
@@ -115,7 +117,7 @@ public class CompositeRolesStore {
                                ReservedRolesStore reservedRolesStore, NativePrivilegeStore privilegeStore,
                                List<BiConsumer<Set<String>, ActionListener<RoleRetrievalResult>>> rolesProviders,
                                ThreadContext threadContext, XPackLicenseState licenseState, FieldPermissionsCache fieldPermissionsCache,
-                               ApiKeyService apiKeyService) {
+                               ApiKeyService apiKeyService, Consumer<Collection<RoleDescriptor>> effectiveRoleDescriptorsConsumer) {
         this.fileRolesStore = fileRolesStore;
         fileRolesStore.addListener(this::invalidate);
         this.nativeRolesStore = nativeRolesStore;
@@ -123,6 +125,7 @@ public class CompositeRolesStore {
         this.licenseState = licenseState;
         this.fieldPermissionsCache = fieldPermissionsCache;
         this.apiKeyService = apiKeyService;
+        this.effectiveRoleDescriptorsConsumer = effectiveRoleDescriptorsConsumer;
         CacheBuilder<RoleKey, Role> builder = CacheBuilder.builder();
         final int cacheSize = CACHE_SIZE_SETTING.get(settings);
         if (cacheSize >= 0) {
@@ -161,9 +164,9 @@ public class CompositeRolesStore {
                     rolesRetrievalResult -> {
                         final boolean missingRoles = rolesRetrievalResult.getMissingRoles().isEmpty() == false;
                         if (missingRoles) {
-                            logger.debug("Could not find roles with names {}", rolesRetrievalResult.getMissingRoles());
+                            logger.debug(() -> new ParameterizedMessage("Could not find roles with names {}",
+                                    rolesRetrievalResult.getMissingRoles()));
                         }
-
                         final Set<RoleDescriptor> effectiveDescriptors;
                         if (licenseState.isDocumentAndFieldLevelSecurityAllowed()) {
                             effectiveDescriptors = rolesRetrievalResult.getRoleDescriptors();
@@ -172,6 +175,11 @@ public class CompositeRolesStore {
                                     .filter((rd) -> rd.isUsingDocumentOrFieldLevelSecurity() == false)
                                     .collect(Collectors.toSet());
                         }
+                        logger.trace(() -> new ParameterizedMessage("Exposing effective role descriptors [{}] for role names [{}]",
+                                effectiveDescriptors, roleNames));
+                        effectiveRoleDescriptorsConsumer.accept(Collections.unmodifiableCollection(effectiveDescriptors));
+                        logger.trace(() -> new ParameterizedMessage("Building role from descriptors [{}] for role names [{}]",
+                                effectiveDescriptors, roleNames));
                         buildThenMaybeCacheRole(roleKey, effectiveDescriptors, rolesRetrievalResult.getMissingRoles(),
                             rolesRetrievalResult.isSuccess(), invalidationCounter, roleActionListener);
                     },
@@ -287,7 +295,7 @@ public class CompositeRolesStore {
     private void roleDescriptors(Set<String> roleNames, ActionListener<RolesRetrievalResult> rolesResultListener) {
         final Set<String> filteredRoleNames = roleNames.stream().filter((s) -> {
             if (negativeLookupCache.get(s) != null) {
-                logger.debug("Requested role [{}] does not exist (cached)", s);
+                logger.debug(() -> new ParameterizedMessage("Requested role [{}] does not exist (cached)", s));
                 return false;
             } else {
                 return true;

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/DeprecationRoleDescriptorConsumer.java

@@ -0,0 +1,222 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security.authz.store;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.util.automaton.Automaton;
+import org.apache.lucene.util.automaton.Operations;
+import org.elasticsearch.cluster.metadata.AliasOrIndex;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.IndicesPrivileges;
+import org.elasticsearch.xpack.core.security.authz.permission.IndicesPermission;
+import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege;
+
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Queue;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.function.Consumer;
+import java.util.function.Predicate;
+
+/**
+ * Inspects all aliases that have greater privileges than the indices that they point to and logs the role descriptor, granting privileges
+ * in this manner, as deprecated and requiring changes. This is done in preparation for the removal of the ability to define privileges over
+ * aliases. The log messages are generated asynchronously and do not generate deprecation response headers. One log entry is generated for
+ * each role descriptor and alias pair, and it contains all the indices for which privileges are a subset of those of the alias. In this
+ * case, the administrator has to adjust the index privileges definition of the respective role such that name patterns do not cover aliases
+ * (or rename aliases). If no logging is generated then the roles used for the current indices and aliases are not vulnerable to the
+ * subsequent breaking change. However, there could be role descriptors that are not used (not mapped to a user that is currently using the
+ * system) which are invisible to this check. Moreover, role descriptors can be dynamically added by role providers. In addition, role
+ * descriptors are merged when building the effective role, so a role-alias pair reported as deprecated might not actually have an impact if
+ * other role descriptors cover its indices. The check iterates over all indices and aliases for each role descriptor so it is quite
+ * expensive computationally. For this reason the check is done only once a day for each role. If the role definitions stay the same, the
+ * deprecations can change from one day to another only if aliases or indices are added.
+ */
+public final class DeprecationRoleDescriptorConsumer implements Consumer<Collection<RoleDescriptor>> {
+
+    private static final String ROLE_PERMISSION_DEPRECATION_STANZA = "Role [%s] contains index privileges covering the [%s] alias but"
+            + " which do not cover some of the indices that it points to [%s]. Granting privileges over an alias and hence granting"
+            + " privileges over all the indices that the alias points to is deprecated and will be removed in a future version of"
+            + " Elasticsearch. Instead define permissions exclusively on index names or index name patterns.";
+
+    private static final Logger logger = LogManager.getLogger(DeprecationRoleDescriptorConsumer.class);
+
+    private final DeprecationLogger deprecationLogger;
+    private final ClusterService clusterService;
+    private final ThreadPool threadPool;
+    private final Object mutex;
+    private final Queue<RoleDescriptor> workQueue;
+    private boolean workerBusy;
+    private final Set<String> dailyRoleCache;
+
+    public DeprecationRoleDescriptorConsumer(ClusterService clusterService, ThreadPool threadPool) {
+        this(clusterService, threadPool, new DeprecationLogger(logger));
+    }
+
+    // package-private for testing
+    DeprecationRoleDescriptorConsumer(ClusterService clusterService, ThreadPool threadPool, DeprecationLogger deprecationLogger) {
+        this.deprecationLogger = deprecationLogger;
+        this.clusterService = clusterService;
+        this.threadPool = threadPool;
+        this.mutex = new Object();
+        this.workQueue = new LinkedList<>();
+        this.workerBusy = false;
+        // this String Set keeps "<date>-<role>" pairs so that we only log a role once a day.
+        this.dailyRoleCache = Collections.newSetFromMap(new LinkedHashMap<String, Boolean>() {
+            @Override
+            protected boolean removeEldestEntry(final Map.Entry<String, Boolean> eldest) {
+                return false == eldest.getKey().startsWith(todayISODate());
+            }
+        });
+    }
+
+    @Override
+    public void accept(Collection<RoleDescriptor> effectiveRoleDescriptors) {
+        synchronized (mutex) {
+            for (RoleDescriptor roleDescriptor : effectiveRoleDescriptors) {
+                if (dailyRoleCache.add(buildCacheKey(roleDescriptor))) {
+                    workQueue.add(roleDescriptor);
+                }
+            }
+            if (false == workerBusy) {
+                workerBusy = true;
+                try {
+                    // spawn another worker on the generic thread pool
+                    threadPool.generic().execute(new AbstractRunnable() {
+
+                        @Override
+                        public void onFailure(Exception e) {
+                            logger.warn("Failed to produce role deprecation messages", e);
+                            synchronized (mutex) {
+                                final boolean hasMoreWork = workQueue.peek() != null;
+                                if (hasMoreWork) {
+                                    workerBusy = true; // just being paranoid :)
+                                    try {
+                                        threadPool.generic().execute(this);
+                                    } catch (RejectedExecutionException e1) {
+                                        workerBusy = false;
+                                        logger.warn("Failed to start working on role alias permisssion deprecation messages", e1);
+                                    }
+                                } else {
+                                    workerBusy = false;
+                                }
+                            }
+                        }
+
+                        @Override
+                        protected void doRun() throws Exception {
+                            while (true) {
+                                final RoleDescriptor workItem;
+                                synchronized (mutex) {
+                                    workItem = workQueue.poll();
+                                    if (workItem == null) {
+                                        workerBusy = false;
+                                        break;
+                                    }
+                                }
+                                logger.trace("Begin role [" + workItem.getName() + "] check for alias permission deprecation");
+                                // executing the check asynchronously will not conserve the generated deprecation response headers (which is
+                                // what we want, because it's not the request that uses deprecated features, but rather the role definition.
+                                // Furthermore, due to caching, we can't reliably associate response headers to every request).
+                                logDeprecatedPermission(workItem);
+                                logger.trace("Completed role [" + workItem.getName() + "] check for alias permission deprecation");
+                            }
+                        }
+                    });
+                } catch (RejectedExecutionException e) {
+                    workerBusy = false;
+                    logger.warn("Failed to start working on role alias permisssion deprecation messages", e);
+                }
+            }
+        }
+    }
+
+    private void logDeprecatedPermission(RoleDescriptor roleDescriptor) {
+        final SortedMap<String, AliasOrIndex> aliasOrIndexMap = clusterService.state().metaData().getAliasAndIndexLookup();
+        final Map<String, Set<String>> privilegesByAliasMap = new HashMap<>();
+        // sort answer by alias for tests
+        final SortedMap<String, Set<String>> privilegesByIndexMap = new TreeMap<>();
+        // collate privileges by index and by alias separately
+        for (final IndicesPrivileges indexPrivilege : roleDescriptor.getIndicesPrivileges()) {
+            final Predicate<String> namePatternPredicate = IndicesPermission.indexMatcher(Arrays.asList(indexPrivilege.getIndices()));
+            for (final Map.Entry<String, AliasOrIndex> aliasOrIndex : aliasOrIndexMap.entrySet()) {
+                final String aliasOrIndexName = aliasOrIndex.getKey();
+                if (namePatternPredicate.test(aliasOrIndexName)) {
+                    if (aliasOrIndex.getValue().isAlias()) {
+                        final Set<String> privilegesByAlias = privilegesByAliasMap.computeIfAbsent(aliasOrIndexName,
+                                k -> new HashSet<String>());
+                        privilegesByAlias.addAll(Arrays.asList(indexPrivilege.getPrivileges()));
+                    } else {
+                        final Set<String> privilegesByIndex = privilegesByIndexMap.computeIfAbsent(aliasOrIndexName,
+                                k -> new HashSet<String>());
+                        privilegesByIndex.addAll(Arrays.asList(indexPrivilege.getPrivileges()));
+                    }
+                }
+            }
+        }
+        // compute privileges Automaton for each alias and for each of the indices it points to
+        final Map<String, Automaton> indexAutomatonMap = new HashMap<>();
+        for (Map.Entry<String, Set<String>> privilegesByAlias : privilegesByAliasMap.entrySet()) {
+            final String aliasName = privilegesByAlias.getKey();
+            final Set<String> aliasPrivilegeNames = privilegesByAlias.getValue();
+            final Automaton aliasPrivilegeAutomaton = IndexPrivilege.get(aliasPrivilegeNames).getAutomaton();
+            final SortedSet<String> inferiorIndexNames = new TreeSet<>();
+            // check if the alias grants superiors privileges than the indices it points to
+            for (IndexMetaData indexMetadata : aliasOrIndexMap.get(aliasName).getIndices()) {
+                final String indexName = indexMetadata.getIndex().getName();
+                final Set<String> indexPrivileges = privilegesByIndexMap.get(indexName);
+                // null iff the index does not have *any* privilege
+                if (indexPrivileges != null) {
+                    // compute automaton once per index no matter how many times it is pointed to
+                    final Automaton indexPrivilegeAutomaton = indexAutomatonMap.computeIfAbsent(indexName,
+                            i -> IndexPrivilege.get(indexPrivileges).getAutomaton());
+                    if (false == Operations.subsetOf(indexPrivilegeAutomaton, aliasPrivilegeAutomaton)) {
+                        inferiorIndexNames.add(indexName);
+                    }
+                } else {
+                    inferiorIndexNames.add(indexName);
+                }
+            }
+            // log inferior indices for this role, for this alias
+            if (false == inferiorIndexNames.isEmpty()) {
+                final String logMessage = String.format(Locale.ROOT, ROLE_PERMISSION_DEPRECATION_STANZA, roleDescriptor.getName(),
+                        aliasName, String.join(", ", inferiorIndexNames));
+                deprecationLogger.deprecated(logMessage);
+            }
+        }
+    }
+
+    private static String todayISODate() {
+        return ZonedDateTime.now(ZoneOffset.UTC).format(DateTimeFormatter.BASIC_ISO_DATE);
+    }
+
+    // package-private for testing
+    static String buildCacheKey(RoleDescriptor roleDescriptor) {
+        return todayISODate() + "-" + roleDescriptor.getName();
+    }
+}

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java

@@ -66,6 +66,7 @@ import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.function.Function;
@@ -74,9 +75,11 @@ import java.util.function.Predicate;
 import static org.elasticsearch.mock.orig.Mockito.times;
 import static org.elasticsearch.mock.orig.Mockito.verifyNoMoreInteractions;
 import static org.hamcrest.Matchers.anyOf;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.nullValue;
+import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItem;
-import static org.hamcrest.Matchers.is;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.anySetOf;
 import static org.mockito.Matchers.eq;
@@ -143,25 +146,35 @@ public class CompositeRolesStoreTests extends ESTestCase {
         when(fileRolesStore.roleDescriptors(Collections.singleton("dls"))).thenReturn(Collections.singleton(dlsRole));
         when(fileRolesStore.roleDescriptors(Collections.singleton("fls_dls"))).thenReturn(Collections.singleton(flsDlsRole));
         when(fileRolesStore.roleDescriptors(Collections.singleton("no_fls_dls"))).thenReturn(Collections.singleton(noFlsDlsRole));
+        final AtomicReference<Collection<RoleDescriptor>> effectiveRoleDescriptors = new AtomicReference<Collection<RoleDescriptor>>();
         CompositeRolesStore compositeRolesStore = new CompositeRolesStore(Settings.EMPTY, fileRolesStore, nativeRolesStore,
                 reservedRolesStore, mock(NativePrivilegeStore.class), Collections.emptyList(),
-                new ThreadContext(Settings.EMPTY), licenseState, cache, mock(ApiKeyService.class));
+                new ThreadContext(Settings.EMPTY), licenseState, cache, mock(ApiKeyService.class),
+                rds -> effectiveRoleDescriptors.set(rds));
 
         PlainActionFuture<Role> roleFuture = new PlainActionFuture<>();
         compositeRolesStore.roles(Collections.singleton("fls"), roleFuture);
         assertEquals(Role.EMPTY, roleFuture.actionGet());
+        assertThat(effectiveRoleDescriptors.get().isEmpty(), is(true));
+        effectiveRoleDescriptors.set(null);
 
         roleFuture = new PlainActionFuture<>();
         compositeRolesStore.roles(Collections.singleton("dls"), roleFuture);
         assertEquals(Role.EMPTY, roleFuture.actionGet());
+        assertThat(effectiveRoleDescriptors.get().isEmpty(), is(true));
+        effectiveRoleDescriptors.set(null);
 
         roleFuture = new PlainActionFuture<>();
         compositeRolesStore.roles(Collections.singleton("fls_dls"), roleFuture);
         assertEquals(Role.EMPTY, roleFuture.actionGet());
+        assertThat(effectiveRoleDescriptors.get().isEmpty(), is(true));
+        effectiveRoleDescriptors.set(null);
 
         roleFuture = new PlainActionFuture<>();
         compositeRolesStore.roles(Collections.singleton("no_fls_dls"), roleFuture);
         assertNotEquals(Role.EMPTY, roleFuture.actionGet());
+        assertThat(effectiveRoleDescriptors.get(), containsInAnyOrder(noFlsDlsRole));
+        effectiveRoleDescriptors.set(null);
     }
 
     public void testRolesWhenDlsFlsLicensed() throws IOException {
@@ -208,25 +221,35 @@ public class CompositeRolesStoreTests extends ESTestCase {
         when(fileRolesStore.roleDescriptors(Collections.singleton("dls"))).thenReturn(Collections.singleton(dlsRole));
         when(fileRolesStore.roleDescriptors(Collections.singleton("fls_dls"))).thenReturn(Collections.singleton(flsDlsRole));
         when(fileRolesStore.roleDescriptors(Collections.singleton("no_fls_dls"))).thenReturn(Collections.singleton(noFlsDlsRole));
+        final AtomicReference<Collection<RoleDescriptor>> effectiveRoleDescriptors = new AtomicReference<Collection<RoleDescriptor>>();
         CompositeRolesStore compositeRolesStore = new CompositeRolesStore(Settings.EMPTY, fileRolesStore, nativeRolesStore,
                 reservedRolesStore, mock(NativePrivilegeStore.class), Collections.emptyList(),
-                new ThreadContext(Settings.EMPTY), licenseState, cache, mock(ApiKeyService.class));
+                new ThreadContext(Settings.EMPTY), licenseState, cache, mock(ApiKeyService.class),
+                rds -> effectiveRoleDescriptors.set(rds));
 
         PlainActionFuture<Role> roleFuture = new PlainActionFuture<>();
         compositeRolesStore.roles(Collections.singleton("fls"), roleFuture);
         assertNotEquals(Role.EMPTY, roleFuture.actionGet());
+        assertThat(effectiveRoleDescriptors.get(), containsInAnyOrder(flsRole));
+        effectiveRoleDescriptors.set(null);
 
         roleFuture = new PlainActionFuture<>();
         compositeRolesStore.roles(Collections.singleton("dls"), roleFuture);
         assertNotEquals(Role.EMPTY, roleFuture.actionGet());
+        assertThat(effectiveRoleDescriptors.get(), containsInAnyOrder(dlsRole));
+        effectiveRoleDescriptors.set(null);
 
         roleFuture = new PlainActionFuture<>();
         compositeRolesStore.roles(Collections.singleton("fls_dls"), roleFuture);
         assertNotEquals(Role.EMPTY, roleFuture.actionGet());
+        assertThat(effectiveRoleDescriptors.get(), containsInAnyOrder(flsDlsRole));
+        effectiveRoleDescriptors.set(null);
 
         roleFuture = new PlainActionFuture<>();
         compositeRolesStore.roles(Collections.singleton("no_fls_dls"), roleFuture);
         assertNotEquals(Role.EMPTY, roleFuture.actionGet());
+        assertThat(effectiveRoleDescriptors.get(), containsInAnyOrder(noFlsDlsRole));
+        effectiveRoleDescriptors.set(null);
     }
 
     public void testNegativeLookupsAreCached() {
@@ -249,16 +272,20 @@ public class CompositeRolesStoreTests extends ESTestCase {
             return null;
         }).when(nativePrivilegeStore).getPrivileges(isA(Set.class), isA(Set.class), any(ActionListener.class));
 
+        final AtomicReference<Collection<RoleDescriptor>> effectiveRoleDescriptors = new AtomicReference<Collection<RoleDescriptor>>();
         final CompositeRolesStore compositeRolesStore =
                 new CompositeRolesStore(SECURITY_ENABLED_SETTINGS, fileRolesStore, nativeRolesStore, reservedRolesStore,
                         nativePrivilegeStore, Collections.emptyList(), new ThreadContext(SECURITY_ENABLED_SETTINGS),
-                        new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, mock(ApiKeyService.class));
+                        new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, mock(ApiKeyService.class),
+                        rds -> effectiveRoleDescriptors.set(rds));
         verify(fileRolesStore).addListener(any(Consumer.class)); // adds a listener in ctor
 
         final String roleName = randomAlphaOfLengthBetween(1, 10);
         PlainActionFuture<Role> future = new PlainActionFuture<>();
         compositeRolesStore.roles(Collections.singleton(roleName), future);
         final Role role = future.actionGet();
+        assertThat(effectiveRoleDescriptors.get().isEmpty(), is(true));
+        effectiveRoleDescriptors.set(null);
         assertEquals(Role.EMPTY, role);
         verify(reservedRolesStore).accept(anySetOf(String.class), any(ActionListener.class));
         verify(fileRolesStore).accept(anySetOf(String.class), any(ActionListener.class));
@@ -275,6 +302,12 @@ public class CompositeRolesStoreTests extends ESTestCase {
             future = new PlainActionFuture<>();
             compositeRolesStore.roles(names, future);
             future.actionGet();
+            if (getSuperuserRole && i == 0) {
+                assertThat(effectiveRoleDescriptors.get(), containsInAnyOrder(ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR));
+                effectiveRoleDescriptors.set(null);
+            } else {
+                assertThat(effectiveRoleDescriptors.get(), is(nullValue()));
+            }
         }
 
         if (getSuperuserRole && numberOfTimesToCall > 0) {
@@ -301,15 +334,18 @@ public class CompositeRolesStoreTests extends ESTestCase {
         final Settings settings = Settings.builder().put(SECURITY_ENABLED_SETTINGS)
             .put("xpack.security.authz.store.roles.negative_lookup_cache.max_size", 0)
             .build();
+        final AtomicReference<Collection<RoleDescriptor>> effectiveRoleDescriptors = new AtomicReference<Collection<RoleDescriptor>>();
         final CompositeRolesStore compositeRolesStore = new CompositeRolesStore(settings, fileRolesStore, nativeRolesStore,
             reservedRolesStore, mock(NativePrivilegeStore.class), Collections.emptyList(), new ThreadContext(settings),
-            new XPackLicenseState(settings), cache, mock(ApiKeyService.class));
+            new XPackLicenseState(settings), cache, mock(ApiKeyService.class), rds -> effectiveRoleDescriptors.set(rds));
         verify(fileRolesStore).addListener(any(Consumer.class)); // adds a listener in ctor
 
         final String roleName = randomAlphaOfLengthBetween(1, 10);
         PlainActionFuture<Role> future = new PlainActionFuture<>();
         compositeRolesStore.roles(Collections.singleton(roleName), future);
         final Role role = future.actionGet();
+        assertThat(effectiveRoleDescriptors.get().isEmpty(), is(true));
+        effectiveRoleDescriptors.set(null);
         assertEquals(Role.EMPTY, role);
         verify(reservedRolesStore).accept(anySetOf(String.class), any(ActionListener.class));
         verify(fileRolesStore).accept(anySetOf(String.class), any(ActionListener.class));
@@ -334,16 +370,20 @@ public class CompositeRolesStoreTests extends ESTestCase {
         }).when(nativeRolesStore).getRoleDescriptors(isA(Set.class), any(ActionListener.class));
         final ReservedRolesStore reservedRolesStore = spy(new ReservedRolesStore());
 
+        final AtomicReference<Collection<RoleDescriptor>> effectiveRoleDescriptors = new AtomicReference<Collection<RoleDescriptor>>();
         final CompositeRolesStore compositeRolesStore =
             new CompositeRolesStore(SECURITY_ENABLED_SETTINGS, fileRolesStore, nativeRolesStore, reservedRolesStore,
                 mock(NativePrivilegeStore.class), Collections.emptyList(), new ThreadContext(SECURITY_ENABLED_SETTINGS),
-                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, mock(ApiKeyService.class));
+                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, mock(ApiKeyService.class),
+                rds -> effectiveRoleDescriptors.set(rds));
         verify(fileRolesStore).addListener(any(Consumer.class)); // adds a listener in ctor
 
         final String roleName = randomAlphaOfLengthBetween(1, 10);
         PlainActionFuture<Role> future = new PlainActionFuture<>();
         compositeRolesStore.roles(Collections.singleton(roleName), future);
         final Role role = future.actionGet();
+        assertThat(effectiveRoleDescriptors.get().isEmpty(), is(true));
+        effectiveRoleDescriptors.set(null);
         assertEquals(Role.EMPTY, role);
         verify(reservedRolesStore).accept(anySetOf(String.class), any(ActionListener.class));
         verify(fileRolesStore).accept(anySetOf(String.class), any(ActionListener.class));
@@ -357,6 +397,8 @@ public class CompositeRolesStoreTests extends ESTestCase {
             future = new PlainActionFuture<>();
             compositeRolesStore.roles(names, future);
             future.actionGet();
+            assertThat(effectiveRoleDescriptors.get().isEmpty(), is(true));
+            effectiveRoleDescriptors.set(null);
         }
 
         assertFalse(compositeRolesStore.isValueInNegativeLookupCache(roleName));
@@ -381,17 +423,22 @@ public class CompositeRolesStoreTests extends ESTestCase {
         }).when(nativeRolesStore).getRoleDescriptors(isA(Set.class), any(ActionListener.class));
         final ReservedRolesStore reservedRolesStore = spy(new ReservedRolesStore());
 
+        final RoleDescriptor roleAProvider1 = new RoleDescriptor("roleA", null,
+                    new IndicesPrivileges[] {
+                        IndicesPrivileges.builder().privileges("READ").indices("foo").grantedFields("*").build()
+                    }, null);
         final InMemoryRolesProvider inMemoryProvider1 = spy(new InMemoryRolesProvider((roles) -> {
             Set<RoleDescriptor> descriptors = new HashSet<>();
             if (roles.contains("roleA")) {
-                descriptors.add(new RoleDescriptor("roleA", null,
-                    new IndicesPrivileges[] {
-                        IndicesPrivileges.builder().privileges("READ").indices("foo").grantedFields("*").build()
-                    }, null));
+                descriptors.add(roleAProvider1);
             }
             return RoleRetrievalResult.success(descriptors);
         }));
 
+        final RoleDescriptor roleBProvider2 = new RoleDescriptor("roleB", null,
+                    new IndicesPrivileges[] {
+                        IndicesPrivileges.builder().privileges("READ").indices("bar").grantedFields("*").build()
+                    }, null);
         final InMemoryRolesProvider inMemoryProvider2 = spy(new InMemoryRolesProvider((roles) -> {
             Set<RoleDescriptor> descriptors = new HashSet<>();
             if (roles.contains("roleA")) {
@@ -403,24 +450,24 @@ public class CompositeRolesStoreTests extends ESTestCase {
                     }, null));
             }
             if (roles.contains("roleB")) {
-                descriptors.add(new RoleDescriptor("roleB", null,
-                    new IndicesPrivileges[] {
-                        IndicesPrivileges.builder().privileges("READ").indices("bar").grantedFields("*").build()
-                    }, null));
+                descriptors.add(roleBProvider2);
             }
             return RoleRetrievalResult.success(descriptors);
         }));
 
+        final AtomicReference<Collection<RoleDescriptor>> effectiveRoleDescriptors = new AtomicReference<Collection<RoleDescriptor>>();
         final CompositeRolesStore compositeRolesStore =
                 new CompositeRolesStore(SECURITY_ENABLED_SETTINGS, fileRolesStore, nativeRolesStore, reservedRolesStore,
                                 mock(NativePrivilegeStore.class), Arrays.asList(inMemoryProvider1, inMemoryProvider2),
-                                new ThreadContext(SECURITY_ENABLED_SETTINGS), new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache,
-                    mock(ApiKeyService.class));
+                                new ThreadContext(SECURITY_ENABLED_SETTINGS), new XPackLicenseState(SECURITY_ENABLED_SETTINGS),
+                                cache, mock(ApiKeyService.class), rds -> effectiveRoleDescriptors.set(rds));
 
         final Set<String> roleNames = Sets.newHashSet("roleA", "roleB", "unknown");
         PlainActionFuture<Role> future = new PlainActionFuture<>();
         compositeRolesStore.roles(roleNames, future);
         final Role role = future.actionGet();
+        assertThat(effectiveRoleDescriptors.get(), containsInAnyOrder(roleAProvider1, roleBProvider2));
+        effectiveRoleDescriptors.set(null);
 
         // make sure custom roles providers populate roles correctly
         assertEquals(2, role.indices().groups().length);
@@ -438,6 +485,12 @@ public class CompositeRolesStoreTests extends ESTestCase {
             future = new PlainActionFuture<>();
             compositeRolesStore.roles(Collections.singleton("unknown"), future);
             future.actionGet();
+            if (i == 0) {
+                assertThat(effectiveRoleDescriptors.get().isEmpty(), is(true));
+            } else {
+                assertThat(effectiveRoleDescriptors.get(), is(nullValue()));
+            }
+            effectiveRoleDescriptors.set(null);
         }
 
         verifyNoMoreInteractions(inMemoryProvider1, inMemoryProvider2);
@@ -616,11 +669,12 @@ public class CompositeRolesStoreTests extends ESTestCase {
         final BiConsumer<Set<String>, ActionListener<RoleRetrievalResult>> failingProvider =
             (roles, listener) -> listener.onFailure(new Exception("fake failure"));
 
+        final AtomicReference<Collection<RoleDescriptor>> effectiveRoleDescriptors = new AtomicReference<Collection<RoleDescriptor>>();
         final CompositeRolesStore compositeRolesStore =
             new CompositeRolesStore(SECURITY_ENABLED_SETTINGS, fileRolesStore, nativeRolesStore, reservedRolesStore,
                 mock(NativePrivilegeStore.class), Arrays.asList(inMemoryProvider1, failingProvider),
-                new ThreadContext(SECURITY_ENABLED_SETTINGS), new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache,
-                mock(ApiKeyService.class));
+                new ThreadContext(SECURITY_ENABLED_SETTINGS), new XPackLicenseState(SECURITY_ENABLED_SETTINGS),
+                cache, mock(ApiKeyService.class), rds -> effectiveRoleDescriptors.set(rds));
 
         final Set<String> roleNames = Sets.newHashSet("roleA", "roleB", "unknown");
         PlainActionFuture<Role> future = new PlainActionFuture<>();
@@ -630,6 +684,7 @@ public class CompositeRolesStoreTests extends ESTestCase {
             fail("provider should have thrown a failure");
         } catch (ExecutionException e) {
             assertEquals("fake failure", e.getCause().getMessage());
+            assertThat(effectiveRoleDescriptors.get(), is(nullValue()));
         }
     }
 
@@ -646,13 +701,14 @@ public class CompositeRolesStoreTests extends ESTestCase {
         }).when(nativeRolesStore).getRoleDescriptors(isA(Set.class), any(ActionListener.class));
         final ReservedRolesStore reservedRolesStore = new ReservedRolesStore();
 
+        final RoleDescriptor roleA = new RoleDescriptor("roleA", null,
+                    new IndicesPrivileges[] {
+                        IndicesPrivileges.builder().privileges("READ").indices("foo").grantedFields("*").build()
+                    }, null);
         final InMemoryRolesProvider inMemoryProvider = new InMemoryRolesProvider((roles) -> {
             Set<RoleDescriptor> descriptors = new HashSet<>();
             if (roles.contains("roleA")) {
-                descriptors.add(new RoleDescriptor("roleA", null,
-                    new IndicesPrivileges[] {
-                        IndicesPrivileges.builder().privileges("READ").indices("foo").grantedFields("*").build()
-                    }, null));
+                descriptors.add(roleA);
             }
             return RoleRetrievalResult.success(descriptors);
         });
@@ -660,27 +716,34 @@ public class CompositeRolesStoreTests extends ESTestCase {
         UpdatableLicenseState xPackLicenseState = new UpdatableLicenseState(SECURITY_ENABLED_SETTINGS);
         // these licenses don't allow custom role providers
         xPackLicenseState.update(randomFrom(OperationMode.BASIC, OperationMode.GOLD, OperationMode.STANDARD), true, null);
+        final AtomicReference<Collection<RoleDescriptor>> effectiveRoleDescriptors = new AtomicReference<Collection<RoleDescriptor>>();
         CompositeRolesStore compositeRolesStore = new CompositeRolesStore(
             Settings.EMPTY, fileRolesStore, nativeRolesStore, reservedRolesStore, mock(NativePrivilegeStore.class),
-            Arrays.asList(inMemoryProvider), new ThreadContext(Settings.EMPTY), xPackLicenseState, cache, mock(ApiKeyService.class));
+            Arrays.asList(inMemoryProvider), new ThreadContext(Settings.EMPTY), xPackLicenseState, cache,
+            mock(ApiKeyService.class), rds -> effectiveRoleDescriptors.set(rds));
 
         Set<String> roleNames = Sets.newHashSet("roleA");
         PlainActionFuture<Role> future = new PlainActionFuture<>();
         compositeRolesStore.roles(roleNames, future);
         Role role = future.actionGet();
+        assertThat(effectiveRoleDescriptors.get().isEmpty(), is(true));
+        effectiveRoleDescriptors.set(null);
 
         // no roles should've been populated, as the license doesn't permit custom role providers
         assertEquals(0, role.indices().groups().length);
 
         compositeRolesStore = new CompositeRolesStore(
             Settings.EMPTY, fileRolesStore, nativeRolesStore, reservedRolesStore, mock(NativePrivilegeStore.class),
-            Arrays.asList(inMemoryProvider), new ThreadContext(Settings.EMPTY), xPackLicenseState, cache, mock(ApiKeyService.class));
+            Arrays.asList(inMemoryProvider), new ThreadContext(Settings.EMPTY), xPackLicenseState, cache,
+            mock(ApiKeyService.class), rds -> effectiveRoleDescriptors.set(rds));
         // these licenses allow custom role providers
         xPackLicenseState.update(randomFrom(OperationMode.PLATINUM, OperationMode.TRIAL), true, null);
         roleNames = Sets.newHashSet("roleA");
         future = new PlainActionFuture<>();
         compositeRolesStore.roles(roleNames, future);
         role = future.actionGet();
+        assertThat(effectiveRoleDescriptors.get(), containsInAnyOrder(roleA));
+        effectiveRoleDescriptors.set(null);
 
         // roleA should've been populated by the custom role provider, because the license allows it
         assertEquals(1, role.indices().groups().length);
@@ -688,13 +751,15 @@ public class CompositeRolesStoreTests extends ESTestCase {
         // license expired, don't allow custom role providers
         compositeRolesStore = new CompositeRolesStore(
             Settings.EMPTY, fileRolesStore, nativeRolesStore, reservedRolesStore, mock(NativePrivilegeStore.class),
-            Arrays.asList(inMemoryProvider), new ThreadContext(Settings.EMPTY), xPackLicenseState, cache, mock(ApiKeyService.class));
+            Arrays.asList(inMemoryProvider), new ThreadContext(Settings.EMPTY), xPackLicenseState, cache,
+            mock(ApiKeyService.class), rds -> effectiveRoleDescriptors.set(rds));
         xPackLicenseState.update(randomFrom(OperationMode.PLATINUM, OperationMode.TRIAL), false, null);
         roleNames = Sets.newHashSet("roleA");
         future = new PlainActionFuture<>();
         compositeRolesStore.roles(roleNames, future);
         role = future.actionGet();
         assertEquals(0, role.indices().groups().length);
+        assertThat(effectiveRoleDescriptors.get().isEmpty(), is(true));
     }
 
     private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) {
@@ -713,7 +778,7 @@ public class CompositeRolesStoreTests extends ESTestCase {
         CompositeRolesStore compositeRolesStore = new CompositeRolesStore(
                 Settings.EMPTY, fileRolesStore, nativeRolesStore, reservedRolesStore,
                 mock(NativePrivilegeStore.class), Collections.emptyList(), new ThreadContext(Settings.EMPTY),
-                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, mock(ApiKeyService.class)) {
+                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, mock(ApiKeyService.class), rds -> {}) {
             @Override
             public void invalidateAll() {
                 numInvalidation.incrementAndGet();
@@ -765,7 +830,7 @@ public class CompositeRolesStoreTests extends ESTestCase {
         CompositeRolesStore compositeRolesStore = new CompositeRolesStore(SECURITY_ENABLED_SETTINGS,
                 fileRolesStore, nativeRolesStore, reservedRolesStore,
                 mock(NativePrivilegeStore.class), Collections.emptyList(), new ThreadContext(SECURITY_ENABLED_SETTINGS),
-                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, mock(ApiKeyService.class)) {
+                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, mock(ApiKeyService.class), rds -> {}) {
             @Override
             public void invalidateAll() {
                 numInvalidation.incrementAndGet();
@@ -799,10 +864,9 @@ public class CompositeRolesStoreTests extends ESTestCase {
         final CompositeRolesStore compositeRolesStore =
             new CompositeRolesStore(SECURITY_ENABLED_SETTINGS, fileRolesStore, nativeRolesStore, reservedRolesStore,
                 mock(NativePrivilegeStore.class), Collections.emptyList(), new ThreadContext(SECURITY_ENABLED_SETTINGS),
-                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, mock(ApiKeyService.class));
+                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, mock(ApiKeyService.class), rds -> {});
         verify(fileRolesStore).addListener(any(Consumer.class)); // adds a listener in ctor
 
-
         PlainActionFuture<Role> rolesFuture = new PlainActionFuture<>();
         final User user = new User("no role user");
         Authentication auth = new Authentication(user, new RealmRef("name", "type", "node"), null);
@@ -839,7 +903,7 @@ public class CompositeRolesStoreTests extends ESTestCase {
         final CompositeRolesStore compositeRolesStore =
             new CompositeRolesStore(settings, fileRolesStore, nativeRolesStore, reservedRolesStore,
                 mock(NativePrivilegeStore.class), Collections.emptyList(), new ThreadContext(settings),
-                new XPackLicenseState(settings), cache, mock(ApiKeyService.class));
+                new XPackLicenseState(settings), cache, mock(ApiKeyService.class), rds -> {});
         verify(fileRolesStore).addListener(any(Consumer.class)); // adds a listener in ctor
 
         PlainActionFuture<Role> rolesFuture = new PlainActionFuture<>();
@@ -863,10 +927,12 @@ public class CompositeRolesStoreTests extends ESTestCase {
         }).when(nativeRolesStore).getRoleDescriptors(isA(Set.class), any(ActionListener.class));
         final ReservedRolesStore reservedRolesStore = spy(new ReservedRolesStore());
 
+        final AtomicReference<Collection<RoleDescriptor>> effectiveRoleDescriptors = new AtomicReference<Collection<RoleDescriptor>>();
         final CompositeRolesStore compositeRolesStore =
             new CompositeRolesStore(SECURITY_ENABLED_SETTINGS, fileRolesStore, nativeRolesStore, reservedRolesStore,
                 mock(NativePrivilegeStore.class), Collections.emptyList(), new ThreadContext(SECURITY_ENABLED_SETTINGS),
-                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, mock(ApiKeyService.class));
+                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, mock(ApiKeyService.class),
+                rds -> effectiveRoleDescriptors.set(rds));
         verify(fileRolesStore).addListener(any(Consumer.class)); // adds a listener in ctor
 
         PlainActionFuture<Role> rolesFuture = new PlainActionFuture<>();
@@ -874,6 +940,7 @@ public class CompositeRolesStoreTests extends ESTestCase {
         compositeRolesStore.getRoles(XPackUser.INSTANCE, auth, rolesFuture);
         final Role roles = rolesFuture.actionGet();
         assertThat(roles, equalTo(XPackUser.ROLE));
+        assertThat(effectiveRoleDescriptors.get(), is(nullValue()));
         verifyNoMoreInteractions(fileRolesStore, nativeRolesStore, reservedRolesStore);
     }
 
@@ -890,13 +957,16 @@ public class CompositeRolesStoreTests extends ESTestCase {
         }).when(nativeRolesStore).getRoleDescriptors(isA(Set.class), any(ActionListener.class));
         final ReservedRolesStore reservedRolesStore = spy(new ReservedRolesStore());
 
+        final AtomicReference<Collection<RoleDescriptor>> effectiveRoleDescriptors = new AtomicReference<Collection<RoleDescriptor>>();
         final CompositeRolesStore compositeRolesStore =
             new CompositeRolesStore(SECURITY_ENABLED_SETTINGS, fileRolesStore, nativeRolesStore, reservedRolesStore,
                 mock(NativePrivilegeStore.class), Collections.emptyList(), new ThreadContext(SECURITY_ENABLED_SETTINGS),
-                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, mock(ApiKeyService.class));
+                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, mock(ApiKeyService.class),
+                rds -> effectiveRoleDescriptors.set(rds));
         verify(fileRolesStore).addListener(any(Consumer.class)); // adds a listener in ctor
         IllegalArgumentException iae = expectThrows(IllegalArgumentException.class,
             () -> compositeRolesStore.getRoles(SystemUser.INSTANCE, null, null));
+        assertThat(effectiveRoleDescriptors.get(), is(nullValue()));
         assertEquals("the user [_system] is the system user and we should never try to get its roles", iae.getMessage());
     }
 
@@ -921,10 +991,13 @@ public class CompositeRolesStoreTests extends ESTestCase {
             listener.onResponse(Collections.emptyList());
             return Void.TYPE;
         }).when(nativePrivStore).getPrivileges(any(Collection.class), any(Collection.class), any(ActionListener.class));
+
+        final AtomicReference<Collection<RoleDescriptor>> effectiveRoleDescriptors = new AtomicReference<Collection<RoleDescriptor>>();
         final CompositeRolesStore compositeRolesStore =
             new CompositeRolesStore(SECURITY_ENABLED_SETTINGS, fileRolesStore, nativeRolesStore, reservedRolesStore,
                 nativePrivStore, Collections.emptyList(), new ThreadContext(SECURITY_ENABLED_SETTINGS),
-                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, apiKeyService);
+                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, apiKeyService,
+                rds -> effectiveRoleDescriptors.set(rds));
         AuditUtil.getOrGenerateRequestId(threadContext);
         final Authentication authentication = new Authentication(new User("test api key user", "superuser"),
             new RealmRef("_es_api_key", "_es_api_key", "node"), null, Version.CURRENT, AuthenticationType.API_KEY, Collections.emptyMap());
@@ -938,7 +1011,7 @@ public class CompositeRolesStoreTests extends ESTestCase {
         PlainActionFuture<Role> roleFuture = new PlainActionFuture<>();
         compositeRolesStore.getRoles(authentication.getUser(), authentication, roleFuture);
         roleFuture.actionGet();
-
+        assertThat(effectiveRoleDescriptors.get(), is(nullValue()));
         verify(apiKeyService).getRoleForApiKey(eq(authentication), any(ActionListener.class));
     }
 
@@ -963,10 +1036,13 @@ public class CompositeRolesStoreTests extends ESTestCase {
             listener.onResponse(Collections.emptyList());
             return Void.TYPE;
         }).when(nativePrivStore).getPrivileges(any(Collection.class), any(Collection.class), any(ActionListener.class));
+
+        final AtomicReference<Collection<RoleDescriptor>> effectiveRoleDescriptors = new AtomicReference<Collection<RoleDescriptor>>();
         final CompositeRolesStore compositeRolesStore =
             new CompositeRolesStore(SECURITY_ENABLED_SETTINGS, fileRolesStore, nativeRolesStore, reservedRolesStore,
                 nativePrivStore, Collections.emptyList(), new ThreadContext(SECURITY_ENABLED_SETTINGS),
-                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, apiKeyService);
+                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, apiKeyService,
+                rds -> effectiveRoleDescriptors.set(rds));
         AuditUtil.getOrGenerateRequestId(threadContext);
         final Authentication authentication = new Authentication(new User("test api key user", "api_key"),
             new RealmRef("_es_api_key", "_es_api_key", "node"), null, Version.CURRENT, AuthenticationType.API_KEY, Collections.emptyMap());
@@ -983,6 +1059,7 @@ public class CompositeRolesStoreTests extends ESTestCase {
         compositeRolesStore.getRoles(authentication.getUser(), authentication, roleFuture);
         Role role = roleFuture.actionGet();
         assertThat(role.checkClusterAction("cluster:admin/foo", Empty.INSTANCE), is(false));
+        assertThat(effectiveRoleDescriptors.get(), is(nullValue()));
         verify(apiKeyService).getRoleForApiKey(eq(authentication), any(ActionListener.class));
     }
 

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/DeprecationRoleDescriptorConsumerTests.java

@@ -0,0 +1,304 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security.authz.store;
+
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.AliasMetaData;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.mock.orig.Mockito;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.junit.Before;
+import org.mockito.stubbing.Answer;
+
+import java.util.Arrays;
+import java.util.concurrent.ExecutorService;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+
+public final class DeprecationRoleDescriptorConsumerTests extends ESTestCase {
+
+    private ThreadPool threadPool;
+
+    @Before
+    public void init() throws Exception {
+        this.threadPool = mock(ThreadPool.class);
+        ExecutorService executorService = mock(ExecutorService.class);
+        Mockito.doAnswer((Answer) invocation -> {
+            final Runnable arg0 = (Runnable) invocation.getArguments()[0];
+            arg0.run();
+            return null;
+        }).when(executorService).execute(Mockito.isA(Runnable.class));
+        when(threadPool.generic()).thenReturn(executorService);
+        when(threadPool.getThreadContext()).thenReturn(new ThreadContext(Settings.EMPTY));
+    }
+
+    public void testSimpleAliasAndIndexPair() throws Exception {
+        final DeprecationLogger deprecationLogger = mock(DeprecationLogger.class);
+        final MetaData.Builder metaDataBuilder = MetaData.builder();
+        addIndex(metaDataBuilder, "index", "alias");
+        final RoleDescriptor roleOverAlias = new RoleDescriptor("roleOverAlias", new String[] { "read" },
+                new RoleDescriptor.IndicesPrivileges[] { indexPrivileges(randomFrom("read", "write", "delete", "index"), "alias") }, null);
+        final RoleDescriptor roleOverIndex = new RoleDescriptor("roleOverIndex", new String[] { "manage" },
+                new RoleDescriptor.IndicesPrivileges[] { indexPrivileges(randomFrom("read", "write", "delete", "index"), "index") }, null);
+        DeprecationRoleDescriptorConsumer deprecationConsumer = new DeprecationRoleDescriptorConsumer(
+                mockClusterService(metaDataBuilder.build()), threadPool, deprecationLogger);
+        deprecationConsumer.accept(Arrays.asList(roleOverAlias, roleOverIndex));
+        verifyLogger(deprecationLogger, "roleOverAlias", "alias", "index");
+        verifyNoMoreInteractions(deprecationLogger);
+    }
+
+    public void testRoleGrantsOnIndexAndAliasPair() throws Exception {
+        final DeprecationLogger deprecationLogger = mock(DeprecationLogger.class);
+        final MetaData.Builder metaDataBuilder = MetaData.builder();
+        addIndex(metaDataBuilder, "index", "alias");
+        addIndex(metaDataBuilder, "index1", "alias2");
+        final RoleDescriptor roleOverIndexAndAlias = new RoleDescriptor("roleOverIndexAndAlias", new String[] { "manage_watcher" },
+                new RoleDescriptor.IndicesPrivileges[] {
+                        indexPrivileges(randomFrom("read", "write", "delete", "index"), "index", "alias") },
+                null);
+        DeprecationRoleDescriptorConsumer deprecationConsumer = new DeprecationRoleDescriptorConsumer(
+                mockClusterService(metaDataBuilder.build()), threadPool, deprecationLogger);
+        deprecationConsumer.accept(Arrays.asList(roleOverIndexAndAlias));
+        verifyNoMoreInteractions(deprecationLogger);
+    }
+
+    public void testMultiplePrivilegesLoggedOnce() throws Exception {
+        final DeprecationLogger deprecationLogger = mock(DeprecationLogger.class);
+        final MetaData.Builder metaDataBuilder = MetaData.builder();
+        addIndex(metaDataBuilder, "index", "alias");
+        addIndex(metaDataBuilder, "index2", "alias2");
+        final RoleDescriptor roleOverAlias = new RoleDescriptor("roleOverAlias", new String[] { "manage_watcher" },
+                new RoleDescriptor.IndicesPrivileges[] {
+                        indexPrivileges("write", "alias"),
+                        indexPrivileges("manage_ilm", "alias") },
+                null);
+        DeprecationRoleDescriptorConsumer deprecationConsumer = new DeprecationRoleDescriptorConsumer(
+                mockClusterService(metaDataBuilder.build()), threadPool, deprecationLogger);
+        deprecationConsumer.accept(Arrays.asList(roleOverAlias));
+        verifyLogger(deprecationLogger, "roleOverAlias", "alias", "index");
+        verifyNoMoreInteractions(deprecationLogger);
+    }
+
+    public void testMultiplePrivilegesLoggedForEachAlias() throws Exception {
+        final DeprecationLogger deprecationLogger = mock(DeprecationLogger.class);
+        final MetaData.Builder metaDataBuilder = MetaData.builder();
+        addIndex(metaDataBuilder, "index", "alias", "alias3");
+        addIndex(metaDataBuilder, "index2", "alias2", "alias", "alias4");
+        addIndex(metaDataBuilder, "index3", "alias3", "alias");
+        addIndex(metaDataBuilder, "index4", "alias4", "alias");
+        addIndex(metaDataBuilder, "foo", "bar");
+        final RoleDescriptor roleMultiplePrivileges = new RoleDescriptor("roleMultiplePrivileges", new String[] { "manage_watcher" },
+                new RoleDescriptor.IndicesPrivileges[] {
+                        indexPrivileges("write", "index2", "alias"),
+                        indexPrivileges("read", "alias4"),
+                        indexPrivileges("delete_index", "alias3", "index"),
+                        indexPrivileges("create_index", "alias3", "index3")},
+                null);
+        DeprecationRoleDescriptorConsumer deprecationConsumer = new DeprecationRoleDescriptorConsumer(
+                mockClusterService(metaDataBuilder.build()), threadPool, deprecationLogger);
+        deprecationConsumer.accept(Arrays.asList(roleMultiplePrivileges));
+        verifyLogger(deprecationLogger, "roleMultiplePrivileges", "alias", "index, index3, index4");
+        verifyLogger(deprecationLogger, "roleMultiplePrivileges", "alias4", "index2, index4");
+        verifyNoMoreInteractions(deprecationLogger);
+    }
+
+    public void testPermissionsOverlapping() throws Exception {
+        final DeprecationLogger deprecationLogger = mock(DeprecationLogger.class);
+        final MetaData.Builder metaDataBuilder = MetaData.builder();
+        addIndex(metaDataBuilder, "index1", "alias1", "bar");
+        addIndex(metaDataBuilder, "index2", "alias2", "baz");
+        addIndex(metaDataBuilder, "foo", "bar");
+        final RoleDescriptor roleOverAliasAndIndex = new RoleDescriptor("roleOverAliasAndIndex", new String[] { "read_ilm" },
+                new RoleDescriptor.IndicesPrivileges[] {
+                        indexPrivileges("monitor", "index2", "alias1"),
+                        indexPrivileges("monitor", "index1", "alias2")},
+                null);
+        DeprecationRoleDescriptorConsumer deprecationConsumer = new DeprecationRoleDescriptorConsumer(
+                mockClusterService(metaDataBuilder.build()), threadPool, deprecationLogger);
+        deprecationConsumer.accept(Arrays.asList(roleOverAliasAndIndex));
+        verifyNoMoreInteractions(deprecationLogger);
+    }
+
+    public void testOverlappingAcrossMultipleRoleDescriptors() throws Exception {
+        final DeprecationLogger deprecationLogger = mock(DeprecationLogger.class);
+        final MetaData.Builder metaDataBuilder = MetaData.builder();
+        addIndex(metaDataBuilder, "index1", "alias1", "bar");
+        addIndex(metaDataBuilder, "index2", "alias2", "baz");
+        addIndex(metaDataBuilder, "foo", "bar");
+        final RoleDescriptor role1 = new RoleDescriptor("role1", new String[] { "monitor_watcher" },
+                new RoleDescriptor.IndicesPrivileges[] {
+                        indexPrivileges("monitor", "index2", "alias1")},
+                null);
+        final RoleDescriptor role2 = new RoleDescriptor("role2", new String[] { "read_ccr" },
+                new RoleDescriptor.IndicesPrivileges[] {
+                        indexPrivileges("monitor", "index1", "alias2")},
+                null);
+        final RoleDescriptor role3 = new RoleDescriptor("role3", new String[] { "monitor_ml" },
+                new RoleDescriptor.IndicesPrivileges[] {
+                        indexPrivileges("index", "bar")},
+                null);
+        DeprecationRoleDescriptorConsumer deprecationConsumer = new DeprecationRoleDescriptorConsumer(
+                mockClusterService(metaDataBuilder.build()), threadPool, deprecationLogger);
+        deprecationConsumer.accept(Arrays.asList(role1, role2, role3));
+        verifyLogger(deprecationLogger, "role1", "alias1", "index1");
+        verifyLogger(deprecationLogger, "role2", "alias2", "index2");
+        verifyLogger(deprecationLogger, "role3", "bar", "foo, index1");
+        verifyNoMoreInteractions(deprecationLogger);
+    }
+
+    public void testDailyRoleCaching() throws Exception {
+        final DeprecationLogger deprecationLogger = mock(DeprecationLogger.class);
+        final MetaData.Builder metaDataBuilder = MetaData.builder();
+        addIndex(metaDataBuilder, "index1", "alias1", "far");
+        addIndex(metaDataBuilder, "index2", "alias2", "baz");
+        addIndex(metaDataBuilder, "foo", "bar");
+        final MetaData metaData = metaDataBuilder.build();
+        RoleDescriptor someRole = new RoleDescriptor("someRole", new String[] { "monitor_rollup" },
+                new RoleDescriptor.IndicesPrivileges[] {
+                        indexPrivileges("monitor", "i*", "bar")},
+                null);
+        final DeprecationRoleDescriptorConsumer deprecationConsumer = new DeprecationRoleDescriptorConsumer(mockClusterService(metaData),
+                threadPool, deprecationLogger);
+        final String cacheKeyBefore = DeprecationRoleDescriptorConsumer.buildCacheKey(someRole);
+        deprecationConsumer.accept(Arrays.asList(someRole));
+        verifyLogger(deprecationLogger, "someRole", "bar", "foo");
+        verifyNoMoreInteractions(deprecationLogger);
+        deprecationConsumer.accept(Arrays.asList(someRole));
+        final String cacheKeyAfter = DeprecationRoleDescriptorConsumer.buildCacheKey(someRole);
+        // we don't do this test if it crosses days
+        if (false == cacheKeyBefore.equals(cacheKeyAfter)) {
+            return;
+        }
+        verifyNoMoreInteractions(deprecationLogger);
+        RoleDescriptor differentRoleSameName = new RoleDescriptor("someRole", new String[] { "manage_pipeline" },
+                new RoleDescriptor.IndicesPrivileges[] {
+                        indexPrivileges("write", "i*", "baz")},
+                null);
+        deprecationConsumer.accept(Arrays.asList(differentRoleSameName));
+        final String cacheKeyAfterParty = DeprecationRoleDescriptorConsumer.buildCacheKey(differentRoleSameName);
+        // we don't do this test if it crosses days
+        if (false == cacheKeyBefore.equals(cacheKeyAfterParty)) {
+            return;
+        }
+        verifyNoMoreInteractions(deprecationLogger);
+    }
+
+    public void testWildcards() throws Exception {
+        final DeprecationLogger deprecationLogger = mock(DeprecationLogger.class);
+        final MetaData.Builder metaDataBuilder = MetaData.builder();
+        addIndex(metaDataBuilder, "index", "alias", "alias3");
+        addIndex(metaDataBuilder, "index2", "alias", "alias2", "alias4");
+        addIndex(metaDataBuilder, "index3", "alias", "alias3");
+        addIndex(metaDataBuilder, "index4", "alias", "alias4");
+        addIndex(metaDataBuilder, "foo", "bar", "baz");
+        MetaData metaData = metaDataBuilder.build();
+        final RoleDescriptor roleGlobalWildcard = new RoleDescriptor("roleGlobalWildcard", new String[] { "manage_token" },
+                new RoleDescriptor.IndicesPrivileges[] {
+                        indexPrivileges(randomFrom("write", "delete_index", "read_cross_cluster"), "*")},
+                null);
+        new DeprecationRoleDescriptorConsumer(mockClusterService(metaData), threadPool, deprecationLogger)
+        .accept(Arrays.asList(roleGlobalWildcard));
+        verifyNoMoreInteractions(deprecationLogger);
+        final RoleDescriptor roleGlobalWildcard2 = new RoleDescriptor("roleGlobalWildcard2", new String[] { "manage_index_templates" },
+                new RoleDescriptor.IndicesPrivileges[] {
+                        indexPrivileges(randomFrom("write", "delete_index", "read_cross_cluster"), "i*", "a*")},
+                null);
+        new DeprecationRoleDescriptorConsumer(mockClusterService(metaData), threadPool, deprecationLogger)
+        .accept(Arrays.asList(roleGlobalWildcard2));
+        verifyNoMoreInteractions(deprecationLogger);
+        final RoleDescriptor roleWildcardOnIndices = new RoleDescriptor("roleWildcardOnIndices", new String[] { "manage_watcher" },
+                new RoleDescriptor.IndicesPrivileges[] {
+                        indexPrivileges("write", "index*", "alias", "alias3"),
+                        indexPrivileges("read", "foo")},
+                null);
+        new DeprecationRoleDescriptorConsumer(mockClusterService(metaData), threadPool, deprecationLogger)
+                .accept(Arrays.asList(roleWildcardOnIndices));
+        verifyNoMoreInteractions(deprecationLogger);
+        final RoleDescriptor roleWildcardOnAliases = new RoleDescriptor("roleWildcardOnAliases", new String[] { "manage_watcher" },
+                new RoleDescriptor.IndicesPrivileges[] {
+                        indexPrivileges("write", "alias*", "index", "index3"),
+                        indexPrivileges("read", "foo", "index2")},
+                null);
+        new DeprecationRoleDescriptorConsumer(mockClusterService(metaData), threadPool, deprecationLogger)
+                .accept(Arrays.asList(roleWildcardOnAliases));
+        verifyLogger(deprecationLogger, "roleWildcardOnAliases", "alias", "index2, index4");
+        verifyLogger(deprecationLogger, "roleWildcardOnAliases", "alias2", "index2");
+        verifyLogger(deprecationLogger, "roleWildcardOnAliases", "alias4", "index2, index4");
+        verifyNoMoreInteractions(deprecationLogger);
+    }
+
+    public void testMultipleIndicesSameAlias() throws Exception {
+        final DeprecationLogger deprecationLogger = mock(DeprecationLogger.class);
+        final MetaData.Builder metaDataBuilder = MetaData.builder();
+        addIndex(metaDataBuilder, "index1", "alias1");
+        addIndex(metaDataBuilder, "index2", "alias1", "alias2");
+        addIndex(metaDataBuilder, "index3", "alias2");
+        final RoleDescriptor roleOverAliasAndIndex = new RoleDescriptor("roleOverAliasAndIndex", new String[] { "manage_ml" },
+                new RoleDescriptor.IndicesPrivileges[] {
+                        indexPrivileges("delete_index", "alias1", "index1") },
+                null);
+        DeprecationRoleDescriptorConsumer deprecationConsumer = new DeprecationRoleDescriptorConsumer(
+                mockClusterService(metaDataBuilder.build()), threadPool, deprecationLogger);
+        deprecationConsumer.accept(Arrays.asList(roleOverAliasAndIndex));
+        verifyLogger(deprecationLogger, "roleOverAliasAndIndex", "alias1", "index2");
+        verifyNoMoreInteractions(deprecationLogger);
+        final RoleDescriptor roleOverAliases = new RoleDescriptor("roleOverAliases", new String[] { "manage_security" },
+                new RoleDescriptor.IndicesPrivileges[] {
+                        indexPrivileges("monitor", "alias1", "alias2") },
+                null);
+        deprecationConsumer.accept(Arrays.asList(roleOverAliases));
+        verifyLogger(deprecationLogger, "roleOverAliases", "alias1", "index1, index2");
+        verifyLogger(deprecationLogger, "roleOverAliases", "alias2", "index2, index3");
+        verifyNoMoreInteractions(deprecationLogger);
+    }
+
+    private void addIndex(MetaData.Builder metaDataBuilder, String index, String... aliases) {
+        final IndexMetaData.Builder indexMetaDataBuilder = IndexMetaData.builder(index)
+                .settings(Settings.builder().put("index.version.created", VersionUtils.randomVersion(random())))
+                .numberOfShards(1)
+                .numberOfReplicas(1);
+        for (final String alias : aliases) {
+            indexMetaDataBuilder.putAlias(AliasMetaData.builder(alias).build());
+        }
+        metaDataBuilder.put(indexMetaDataBuilder.build(), false);
+    }
+
+    private ClusterService mockClusterService(MetaData metaData) {
+        final ClusterService clusterService = mock(ClusterService.class);
+        final ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).build();
+        when(clusterService.state()).thenReturn(clusterState);
+        return clusterService;
+    }
+
+    private RoleDescriptor.IndicesPrivileges indexPrivileges(String priv, String... indicesOrAliases) {
+        return RoleDescriptor.IndicesPrivileges.builder()
+                .indices(indicesOrAliases)
+                .privileges(priv)
+                .grantedFields(randomArray(0, 2, String[]::new, () -> randomBoolean() ? null : randomAlphaOfLengthBetween(1, 4)))
+                .query(randomBoolean() ? null : "{ }")
+                .build();
+    }
+
+    private void verifyLogger(DeprecationLogger deprecationLogger, String roleName, String aliasName, String indexNames) {
+        verify(deprecationLogger).deprecated("Role [" + roleName + "] contains index privileges covering the [" + aliasName
+                + "] alias but which do not cover some of the indices that it points to [" + indexNames + "]. Granting privileges over an"
+                + " alias and hence granting privileges over all the indices that the alias points to is deprecated and will be removed"
+                + " in a future version of Elasticsearch. Instead define permissions exclusively on index names or index name patterns.");
+    }
+}