Fix cache invalidation on privilege modification (#102193)

Previously the application privilege cache would only be updated for
privileges that were _created_. If an existing privilge was modified,
then that would not automatically invalidate the cache.

This was because the NativePrivilegeStore.putPrivileges had a confusing
response type that replied with the set of privileges that were created,
and left the caller to assume that any other privileges in the request
were updated.

This commit changes the response type of that method to return all
privileges that were included in the request, along with their Result

Relates: #102056

docs/changelog/102193.yaml

@@ -0,0 +1,5 @@
+pr: 102193
+summary: Fix cache invalidation on privilege modification
+area: Authorization
+type: bug
+issues: []

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportPutPrivilegesAction.java

@@ -7,9 +7,11 @@
 package org.elasticsearch.xpack.security.action.privilege;
 
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
@@ -19,6 +21,8 @@ import org.elasticsearch.xpack.core.security.action.privilege.PutPrivilegesRespo
 import org.elasticsearch.xpack.security.authz.store.NativePrivilegeStore;
 
 import java.util.Collections;
+import java.util.List;
+import java.util.Map;
 
 /**
  * Transport action to retrieve one or more application privileges from the security index
@@ -45,8 +49,23 @@ public class TransportPutPrivilegesAction extends HandledTransportAction<PutPriv
             this.privilegeStore.putPrivileges(
                 request.getPrivileges(),
                 request.getRefreshPolicy(),
-                ActionListener.wrap(created -> listener.onResponse(new PutPrivilegesResponse(created)), listener::onFailure)
+                ActionListener.wrap(result -> listener.onResponse(buildResponse(result)), listener::onFailure)
             );
         }
     }
+
+    private static PutPrivilegesResponse buildResponse(Map<String, Map<String, DocWriteResponse.Result>> result) {
+        final Map<String, List<String>> createdPrivilegesByApplicationName = Maps.newHashMapWithExpectedSize(result.size());
+        result.forEach((appName, privileges) -> {
+            List<String> createdPrivileges = privileges.entrySet()
+                .stream()
+                .filter(e -> e.getValue() == DocWriteResponse.Result.CREATED)
+                .map(e -> e.getKey())
+                .toList();
+            if (createdPrivileges.isEmpty() == false) {
+                createdPrivilegesByApplicationName.put(appName, createdPrivileges);
+            }
+        });
+        return new PutPrivilegesResponse(createdPrivilegesByApplicationName);
+    }
 }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java

@@ -370,7 +370,7 @@ public class NativePrivilegeStore {
     public void putPrivileges(
         Collection<ApplicationPrivilegeDescriptor> privileges,
         WriteRequest.RefreshPolicy refreshPolicy,
-        ActionListener<Map<String, List<String>>> listener
+        ActionListener<Map<String, Map<String, DocWriteResponse.Result>>> listener
     ) {
         if (privileges.isEmpty()) {
             listener.onResponse(Map.of());
@@ -416,9 +416,9 @@ public class NativePrivilegeStore {
         }
     }
 
-    private void handleBulkResponse(BulkResponse bulkResponse, ActionListener<Map<String, List<String>>> listener) {
+    private void handleBulkResponse(BulkResponse bulkResponse, ActionListener<Map<String, Map<String, DocWriteResponse.Result>>> listener) {
         ElasticsearchException failure = null;
-        final Map<String, List<String>> createdPrivilegesByAppName = new HashMap<>();
+        final Map<String, Map<String, DocWriteResponse.Result>> privilegeResultByAppName = new HashMap<>();
         for (var item : bulkResponse.getItems()) {
             if (item.isFailed()) {
                 if (failure == null) {
@@ -427,24 +427,22 @@ public class NativePrivilegeStore {
                     failure.addSuppressed(item.getFailure().getCause());
                 }
             } else {
-                if (item.getResponse().getResult() == DocWriteResponse.Result.CREATED) {
-                    final Tuple<String, String> name = nameFromDocId(item.getId());
-                    final String appName = name.v1();
-                    final String privilegeName = name.v2();
-
-                    List<String> createdPrivileges = createdPrivilegesByAppName.get(appName);
-                    if (createdPrivileges == null) {
-                        createdPrivileges = new ArrayList<>();
-                        createdPrivilegesByAppName.put(appName, createdPrivileges);
-                    }
-                    createdPrivileges.add(privilegeName);
+                final Tuple<String, String> name = nameFromDocId(item.getId());
+                final String appName = name.v1();
+                final String privilegeName = name.v2();
+
+                var privileges = privilegeResultByAppName.get(appName);
+                if (privileges == null) {
+                    privileges = new HashMap<>();
+                    privilegeResultByAppName.put(appName, privileges);
                 }
+                privileges.put(privilegeName, item.getResponse().getResult());
             }
         }
         if (failure != null) {
             listener.onFailure(failure);
         } else {
-            clearCaches(listener, createdPrivilegesByAppName.keySet(), createdPrivilegesByAppName);
+            clearCaches(listener, privilegeResultByAppName.keySet(), privilegeResultByAppName);
         }
     }
 

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java

@@ -12,6 +12,7 @@ import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.bulk.BulkItemResponse;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkResponse;
@@ -46,6 +47,7 @@ import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.core.security.action.privilege.ClearPrivilegesCacheRequest;
+import org.elasticsearch.xpack.core.security.action.privilege.ClearPrivilegesCacheResponse;
 import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege;
 import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
 import org.elasticsearch.xpack.core.security.test.TestRestrictedIndices;
@@ -71,6 +73,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
+import java.util.function.Predicate;
 
 import static java.util.Collections.emptyList;
 import static java.util.Collections.emptyMap;
@@ -79,15 +82,20 @@ import static java.util.Collections.singleton;
 import static java.util.Collections.singletonList;
 import static org.elasticsearch.common.util.set.Sets.newHashSet;
 import static org.elasticsearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES;
+import static org.hamcrest.Matchers.aMapWithSize;
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.arrayContaining;
-import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.everyItem;
+import static org.hamcrest.Matchers.hasEntry;
+import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.iterableWithSize;
 import static org.hamcrest.Matchers.not;
 import static org.mockito.ArgumentMatchers.any;
@@ -173,8 +181,7 @@ public class NativePrivilegeStoreTests extends ESTestCase {
         final PlainActionFuture<Collection<ApplicationPrivilegeDescriptor>> future = new PlainActionFuture<>();
         store.getPrivileges(List.of("myapp"), List.of("admin"), future);
         assertThat(requests, iterableWithSize(1));
-        assertThat(requests.get(0), instanceOf(SearchRequest.class));
-        SearchRequest request = (SearchRequest) requests.get(0);
+        final SearchRequest request = getLastRequest(SearchRequest.class);
         final String query = Strings.toString(request.source().query());
         assertThat(query, containsString("""
             {"terms":{"application":["myapp"]"""));
@@ -182,27 +189,7 @@ public class NativePrivilegeStoreTests extends ESTestCase {
             {"term":{"type":{"value":"application-privilege\""""));
 
         final SearchHit[] hits = buildHits(sourcePrivileges);
-        listener.get()
-            .onResponse(
-                new SearchResponse(
-                    new SearchResponseSections(
-                        new SearchHits(hits, new TotalHits(hits.length, TotalHits.Relation.EQUAL_TO), 0f),
-                        null,
-                        null,
-                        false,
-                        false,
-                        null,
-                        1
-                    ),
-                    "_scrollId1",
-                    1,
-                    1,
-                    0,
-                    1,
-                    null,
-                    null
-                )
-            );
+        listener.get().onResponse(buildSearchResponse(hits));
 
         assertResult(sourcePrivileges, future);
     }
@@ -211,27 +198,7 @@ public class NativePrivilegeStoreTests extends ESTestCase {
         final PlainActionFuture<Collection<ApplicationPrivilegeDescriptor>> future = new PlainActionFuture<>();
         store.getPrivileges(List.of("myapp"), List.of("admin"), future);
         final SearchHit[] hits = new SearchHit[0];
-        listener.get()
-            .onResponse(
-                new SearchResponse(
-                    new SearchResponseSections(
-                        new SearchHits(hits, new TotalHits(hits.length, TotalHits.Relation.EQUAL_TO), 0f),
-                        null,
-                        null,
-                        false,
-                        false,
-                        null,
-                        1
-                    ),
-                    "_scrollId1",
-                    1,
-                    1,
-                    0,
-                    1,
-                    null,
-                    null
-                )
-            );
+        listener.get().onResponse(buildSearchResponse(hits));
 
         final Collection<ApplicationPrivilegeDescriptor> applicationPrivilegeDescriptors = future.get(1, TimeUnit.SECONDS);
         assertThat(applicationPrivilegeDescriptors, empty());
@@ -247,8 +214,7 @@ public class NativePrivilegeStoreTests extends ESTestCase {
         final PlainActionFuture<Collection<ApplicationPrivilegeDescriptor>> future = new PlainActionFuture<>();
         store.getPrivileges(Arrays.asList("myapp", "yourapp"), null, future);
         assertThat(requests, iterableWithSize(1));
-        assertThat(requests.get(0), instanceOf(SearchRequest.class));
-        SearchRequest request = (SearchRequest) requests.get(0);
+        final SearchRequest request = getLastRequest(SearchRequest.class);
         assertThat(request.indices(), arrayContaining(SecuritySystemIndices.SECURITY_MAIN_ALIAS));
 
         final String query = Strings.toString(request.source().query());
@@ -259,27 +225,7 @@ public class NativePrivilegeStoreTests extends ESTestCase {
             {"term":{"type":{"value":"application-privilege\""""));
 
         final SearchHit[] hits = buildHits(sourcePrivileges);
-        listener.get()
-            .onResponse(
-                new SearchResponse(
-                    new SearchResponseSections(
-                        new SearchHits(hits, new TotalHits(hits.length, TotalHits.Relation.EQUAL_TO), 0f),
-                        null,
-                        null,
-                        false,
-                        false,
-                        null,
-                        1
-                    ),
-                    "_scrollId1",
-                    1,
-                    1,
-                    0,
-                    1,
-                    null,
-                    null
-                )
-            );
+        listener.get().onResponse(buildSearchResponse(hits));
 
         assertResult(sourcePrivileges, future);
     }
@@ -321,8 +267,7 @@ public class NativePrivilegeStoreTests extends ESTestCase {
         final PlainActionFuture<Collection<ApplicationPrivilegeDescriptor>> future = new PlainActionFuture<>();
         store.getPrivileges(Arrays.asList("myapp-*", "yourapp"), null, future);
         assertThat(requests, iterableWithSize(1));
-        assertThat(requests.get(0), instanceOf(SearchRequest.class));
-        SearchRequest request = (SearchRequest) requests.get(0);
+        final SearchRequest request = getLastRequest(SearchRequest.class);
         assertThat(request.indices(), arrayContaining(SecuritySystemIndices.SECURITY_MAIN_ALIAS));
 
         final String query = Strings.toString(request.source().query());
@@ -338,27 +283,7 @@ public class NativePrivilegeStoreTests extends ESTestCase {
         }
 
         final SearchHit[] hits = buildHits(allowExpensiveQueries ? sourcePrivileges.subList(1, 4) : sourcePrivileges);
-        listener.get()
-            .onResponse(
-                new SearchResponse(
-                    new SearchResponseSections(
-                        new SearchHits(hits, new TotalHits(hits.length, TotalHits.Relation.EQUAL_TO), 0f),
-                        null,
-                        null,
-                        false,
-                        false,
-                        null,
-                        1
-                    ),
-                    "_scrollId1",
-                    1,
-                    1,
-                    0,
-                    1,
-                    null,
-                    null
-                )
-            );
+        listener.get().onResponse(buildSearchResponse(hits));
         // The first and last privilege should not be retrieved
         assertResult(sourcePrivileges.subList(1, 4), future);
     }
@@ -367,8 +292,7 @@ public class NativePrivilegeStoreTests extends ESTestCase {
         final PlainActionFuture<Collection<ApplicationPrivilegeDescriptor>> future = new PlainActionFuture<>();
         store.getPrivileges(Arrays.asList("*", "anything"), null, future);
         assertThat(requests, iterableWithSize(1));
-        assertThat(requests.get(0), instanceOf(SearchRequest.class));
-        SearchRequest request = (SearchRequest) requests.get(0);
+        final SearchRequest request = getLastRequest(SearchRequest.class);
         assertThat(request.indices(), arrayContaining(SecuritySystemIndices.SECURITY_MAIN_ALIAS));
 
         final String query = Strings.toString(request.source().query());
@@ -376,27 +300,7 @@ public class NativePrivilegeStoreTests extends ESTestCase {
         assertThat(query, containsString("{\"term\":{\"type\":{\"value\":\"application-privilege\""));
 
         final SearchHit[] hits = new SearchHit[0];
-        listener.get()
-            .onResponse(
-                new SearchResponse(
-                    new SearchResponseSections(
-                        new SearchHits(hits, new TotalHits(hits.length, TotalHits.Relation.EQUAL_TO), 0f),
-                        null,
-                        null,
-                        false,
-                        false,
-                        null,
-                        1
-                    ),
-                    "_scrollId1",
-                    1,
-                    1,
-                    0,
-                    1,
-                    null,
-                    null
-                )
-            );
+        listener.get().onResponse(buildSearchResponse(hits));
     }
 
     public void testGetAllPrivileges() throws Exception {
@@ -409,8 +313,7 @@ public class NativePrivilegeStoreTests extends ESTestCase {
         final PlainActionFuture<Collection<ApplicationPrivilegeDescriptor>> future = new PlainActionFuture<>();
         store.getPrivileges(null, null, future);
         assertThat(requests, iterableWithSize(1));
-        assertThat(requests.get(0), instanceOf(SearchRequest.class));
-        SearchRequest request = (SearchRequest) requests.get(0);
+        final SearchRequest request = getLastRequest(SearchRequest.class);
         assertThat(request.indices(), arrayContaining(SecuritySystemIndices.SECURITY_MAIN_ALIAS));
 
         final String query = Strings.toString(request.source().query());
@@ -418,27 +321,7 @@ public class NativePrivilegeStoreTests extends ESTestCase {
         assertThat(query, not(containsString("{\"terms\"")));
 
         final SearchHit[] hits = buildHits(sourcePrivileges);
-        listener.get()
-            .onResponse(
-                new SearchResponse(
-                    new SearchResponseSections(
-                        new SearchHits(hits, new TotalHits(hits.length, TotalHits.Relation.EQUAL_TO), 0f),
-                        null,
-                        null,
-                        false,
-                        false,
-                        null,
-                        1
-                    ),
-                    "_scrollId1",
-                    1,
-                    1,
-                    0,
-                    1,
-                    null,
-                    null
-                )
-            );
+        listener.get().onResponse(buildSearchResponse(hits));
 
         assertResult(sourcePrivileges, future);
     }
@@ -454,27 +337,7 @@ public class NativePrivilegeStoreTests extends ESTestCase {
         store.getPrivileges(List.of("myapp", "yourapp"), null, future);
 
         final SearchHit[] hits = buildHits(sourcePrivileges);
-        listener.get()
-            .onResponse(
-                new SearchResponse(
-                    new SearchResponseSections(
-                        new SearchHits(hits, new TotalHits(hits.length, TotalHits.Relation.EQUAL_TO), 0f),
-                        null,
-                        null,
-                        false,
-                        false,
-                        null,
-                        1
-                    ),
-                    "_scrollId1",
-                    1,
-                    1,
-                    0,
-                    1,
-                    null,
-                    null
-                )
-            );
+        listener.get().onResponse(buildSearchResponse(hits));
 
         assertEquals(Set.of("myapp"), store.getApplicationNamesCache().get(Set.of("myapp", "yourapp")));
         assertEquals(Set.copyOf(sourcePrivileges), store.getDescriptorsCache().get("myapp"));
@@ -506,27 +369,7 @@ public class NativePrivilegeStoreTests extends ESTestCase {
         store.getPrivileges(Collections.singletonList("myapp"), singletonList("user"), future);
 
         final SearchHit[] hits = buildHits(sourcePrivileges);
-        listener.get()
-            .onResponse(
-                new SearchResponse(
-                    new SearchResponseSections(
-                        new SearchHits(hits, new TotalHits(hits.length, TotalHits.Relation.EQUAL_TO), 0f),
-                        null,
-                        null,
-                        false,
-                        false,
-                        null,
-                        1
-                    ),
-                    "_scrollId1",
-                    1,
-                    1,
-                    0,
-                    1,
-                    null,
-                    null
-                )
-            );
+        listener.get().onResponse(buildSearchResponse(hits));
 
         // Not caching names with no wildcard
         assertNull(store.getApplicationNamesCache().get(singleton("myapp")));
@@ -545,27 +388,7 @@ public class NativePrivilegeStoreTests extends ESTestCase {
         final PlainActionFuture<Collection<ApplicationPrivilegeDescriptor>> future = new PlainActionFuture<>();
         store.getPrivileges(Collections.singletonList("no-such-app"), null, future);
         final SearchHit[] hits = buildHits(emptyList());
-        listener.get()
-            .onResponse(
-                new SearchResponse(
-                    new SearchResponseSections(
-                        new SearchHits(hits, new TotalHits(hits.length, TotalHits.Relation.EQUAL_TO), 0f),
-                        null,
-                        null,
-                        false,
-                        false,
-                        null,
-                        1
-                    ),
-                    "_scrollId1",
-                    1,
-                    1,
-                    0,
-                    1,
-                    null,
-                    null
-                )
-            );
+        listener.get().onResponse(buildSearchResponse(hits));
 
         assertEquals(emptySet(), store.getApplicationNamesCache().get(singleton("no-such-app")));
         assertEquals(0, store.getDescriptorsCache().count());
@@ -582,27 +405,7 @@ public class NativePrivilegeStoreTests extends ESTestCase {
         final PlainActionFuture<Collection<ApplicationPrivilegeDescriptor>> future = new PlainActionFuture<>();
         store.getPrivileges(emptyList(), null, future);
         final SearchHit[] hits = buildHits(emptyList());
-        listener.get()
-            .onResponse(
-                new SearchResponse(
-                    new SearchResponseSections(
-                        new SearchHits(hits, new TotalHits(hits.length, TotalHits.Relation.EQUAL_TO), 0f),
-                        null,
-                        null,
-                        false,
-                        false,
-                        null,
-                        1
-                    ),
-                    "_scrollId1",
-                    1,
-                    1,
-                    0,
-                    1,
-                    null,
-                    null
-                )
-            );
+        listener.get().onResponse(buildSearchResponse(hits));
         assertEquals(emptySet(), store.getApplicationNamesCache().get(singleton("*")));
         assertEquals(1, store.getApplicationNamesCache().count());
         assertResult(emptyList(), future);
@@ -629,6 +432,68 @@ public class NativePrivilegeStoreTests extends ESTestCase {
         assertResult(emptyList(), future4);
     }
 
+    public void testCacheIsClearedByApplicationNameWhenPrivilegesAreModified() throws Exception {
+        final PlainActionFuture<Collection<ApplicationPrivilegeDescriptor>> getFuture = new PlainActionFuture<>();
+        store.getPrivileges(emptyList(), null, getFuture);
+        final List<ApplicationPrivilegeDescriptor> sourcePrivileges = List.of(
+            new ApplicationPrivilegeDescriptor("app1", "priv1a", Set.of("action:1a"), Map.of()),
+            new ApplicationPrivilegeDescriptor("app1", "priv1b", Set.of("action:1b"), Map.of()),
+            new ApplicationPrivilegeDescriptor("app2", "priv2a", Set.of("action:2a"), Map.of()),
+            new ApplicationPrivilegeDescriptor("app2", "priv2b", Set.of("action:2b"), Map.of())
+        );
+        final SearchHit[] hits = buildHits(sourcePrivileges);
+        listener.get().onResponse(buildSearchResponse(hits));
+        assertEquals(Set.of("app1", "app2"), store.getApplicationNamesCache().get(singleton("*")));
+        assertResult(sourcePrivileges, getFuture);
+
+        // add a new privilege to app1
+        var priv1c = new ApplicationPrivilegeDescriptor("app1", "priv1c", Set.of("action:1c"), Map.of());
+        PlainActionFuture<Map<String, Map<String, DocWriteResponse.Result>>> putFuture = new PlainActionFuture<>();
+        store.putPrivileges(List.of(priv1c), WriteRequest.RefreshPolicy.IMMEDIATE, putFuture);
+
+        handleBulkRequest(1, item -> true);
+
+        assertCacheCleared("app1");
+
+        Map<String, Map<String, DocWriteResponse.Result>> putResponse = putFuture.get();
+        assertThat(putResponse, aMapWithSize(1));
+        assertThat(putResponse, hasKey("app1"));
+        assertThat(putResponse.get("app1"), aMapWithSize(1));
+        assertThat(putResponse.get("app1"), hasEntry("priv1c", DocWriteResponse.Result.CREATED));
+
+        // modify a privilege in app2
+        var priv2a = new ApplicationPrivilegeDescriptor("app2", "priv2a", Set.of("action:2*"), Map.of());
+        putFuture = new PlainActionFuture<>();
+        store.putPrivileges(List.of(priv2a), WriteRequest.RefreshPolicy.IMMEDIATE, putFuture);
+
+        handleBulkRequest(1, item -> false);
+        assertCacheCleared("app2");
+
+        putResponse = putFuture.get();
+        assertThat(putResponse, aMapWithSize(1));
+        assertThat(putResponse, hasKey("app2"));
+        assertThat(putResponse.get("app2"), aMapWithSize(1));
+        assertThat(putResponse.get("app2"), hasEntry("priv2a", DocWriteResponse.Result.UPDATED));
+
+        // modify a privilege in app1, add a privilege in app2
+        var priv1a = new ApplicationPrivilegeDescriptor("app1", "priv1a", Set.of("action:1*"), Map.of());
+        var priv2c = new ApplicationPrivilegeDescriptor("app2", "priv2c", Set.of("action:2c"), Map.of());
+        putFuture = new PlainActionFuture<>();
+        store.putPrivileges(List.of(priv1a, priv2c), WriteRequest.RefreshPolicy.IMMEDIATE, putFuture);
+
+        handleBulkRequest(2, item -> item.id().contains("app2"));
+        assertCacheCleared("app1", "app2");
+
+        putResponse = putFuture.get();
+        assertThat(putResponse, aMapWithSize(2));
+        assertThat(putResponse, hasKey("app1"));
+        assertThat(putResponse.get("app1"), aMapWithSize(1));
+        assertThat(putResponse.get("app1"), hasEntry("priv1a", DocWriteResponse.Result.UPDATED));
+        assertThat(putResponse, hasKey("app2"));
+        assertThat(putResponse.get("app2"), aMapWithSize(1));
+        assertThat(putResponse.get("app2"), hasEntry("priv2c", DocWriteResponse.Result.CREATED));
+    }
+
     public void testStaleResultsWillNotBeCached() {
         final List<ApplicationPrivilegeDescriptor> sourcePrivileges = singletonList(
             new ApplicationPrivilegeDescriptor("myapp", "admin", newHashSet("action:admin/*", "action:login", "data:read/*"), emptyMap())
@@ -640,27 +505,7 @@ public class NativePrivilegeStoreTests extends ESTestCase {
         // Before the results can be cached, invalidate the cache to simulate stale search results
         store.getDescriptorsAndApplicationNamesCache().invalidateAll();
         final SearchHit[] hits = buildHits(sourcePrivileges);
-        listener.get()
-            .onResponse(
-                new SearchResponse(
-                    new SearchResponseSections(
-                        new SearchHits(hits, new TotalHits(hits.length, TotalHits.Relation.EQUAL_TO), 0f),
-                        null,
-                        null,
-                        false,
-                        false,
-                        null,
-                        1
-                    ),
-                    "_scrollId1",
-                    1,
-                    1,
-                    0,
-                    1,
-                    null,
-                    null
-                )
-            );
+        listener.get().onResponse(buildSearchResponse(hits));
 
         // Nothing should be cached since the results are stale
         assertEquals(0, store.getApplicationNamesCache().count());
@@ -708,27 +553,7 @@ public class NativePrivilegeStoreTests extends ESTestCase {
         final PlainActionFuture<Collection<ApplicationPrivilegeDescriptor>> future = new PlainActionFuture<>();
         store1.getPrivileges(null, null, future);
         final SearchHit[] hits = buildHits(sourcePrivileges);
-        listener.get()
-            .onResponse(
-                new SearchResponse(
-                    new SearchResponseSections(
-                        new SearchHits(hits, new TotalHits(hits.length, TotalHits.Relation.EQUAL_TO), 0f),
-                        null,
-                        null,
-                        false,
-                        false,
-                        null,
-                        1
-                    ),
-                    "_scrollId1",
-                    1,
-                    1,
-                    0,
-                    1,
-                    null,
-                    null
-                )
-            );
+        listener.get().onResponse(buildSearchResponse(hits));
 
         // Make sure the caching is about to happen
         getPrivilegeCountDown.await(5, TimeUnit.SECONDS);
@@ -750,7 +575,7 @@ public class NativePrivilegeStoreTests extends ESTestCase {
             new ApplicationPrivilegeDescriptor("app2", "all", newHashSet("*"), emptyMap())
         );
 
-        final PlainActionFuture<Map<String, List<String>>> putPrivilegeFuture = new PlainActionFuture<>();
+        final PlainActionFuture<Map<String, Map<String, DocWriteResponse.Result>>> putPrivilegeFuture = new PlainActionFuture<>();
         store.putPrivileges(putPrivileges, WriteRequest.RefreshPolicy.IMMEDIATE, putPrivilegeFuture);
         assertThat(requests, iterableWithSize(1));
         assertThat(requests, everyItem(instanceOf(BulkRequest.class)));
@@ -776,7 +601,7 @@ public class NativePrivilegeStoreTests extends ESTestCase {
             final boolean created = privilege.getName().equals("user") == false;
             responses[i] = BulkItemResponse.success(
                 i,
-                created ? DocWriteRequest.OpType.CREATE : DocWriteRequest.OpType.UPDATE,
+                DocWriteRequest.OpType.INDEX,
                 new IndexResponse(new ShardId(SecuritySystemIndices.SECURITY_MAIN_ALIAS, uuid, i), request.id(), 1, 1, 1, created)
             );
         }
@@ -789,12 +614,13 @@ public class NativePrivilegeStoreTests extends ESTestCase {
         assertThat(requests.get(0), instanceOf(ClearPrivilegesCacheRequest.class));
         listener.get().onResponse(null);
 
-        final Map<String, List<String>> map = putPrivilegeFuture.actionGet();
+        final Map<String, Map<String, DocWriteResponse.Result>> map = putPrivilegeFuture.actionGet();
         assertThat(map.entrySet(), iterableWithSize(2));
-        assertThat(map.get("app1"), iterableWithSize(1));
-        assertThat(map.get("app2"), iterableWithSize(1));
-        assertThat(map.get("app1"), contains("admin"));
-        assertThat(map.get("app2"), contains("all"));
+        assertThat(map.get("app1"), aMapWithSize(2));
+        assertThat(map.get("app2"), aMapWithSize(1));
+        assertThat(map.get("app1"), hasEntry("admin", DocWriteResponse.Result.CREATED));
+        assertThat(map.get("app1"), hasEntry("user", DocWriteResponse.Result.UPDATED));
+        assertThat(map.get("app2"), hasEntry("all", DocWriteResponse.Result.CREATED));
     }
 
     public void testRetrieveActionNamePatternsInsteadOfPrivileges() throws Exception {
@@ -953,27 +779,7 @@ public class NativePrivilegeStoreTests extends ESTestCase {
         final PlainActionFuture<Collection<ApplicationPrivilegeDescriptor>> future = new PlainActionFuture<>();
         store1.getPrivileges(singletonList("myapp"), null, future);
         final SearchHit[] hits = buildHits(sourcePrivileges);
-        listener.get()
-            .onResponse(
-                new SearchResponse(
-                    new SearchResponseSections(
-                        new SearchHits(hits, new TotalHits(hits.length, TotalHits.Relation.EQUAL_TO), 0f),
-                        null,
-                        null,
-                        false,
-                        false,
-                        null,
-                        1
-                    ),
-                    "_scrollId1",
-                    1,
-                    1,
-                    0,
-                    1,
-                    null,
-                    null
-                )
-            );
+        listener.get().onResponse(buildSearchResponse(hits));
 
         assertResult(sourcePrivileges, future);
     }
@@ -998,6 +804,12 @@ public class NativePrivilegeStoreTests extends ESTestCase {
         );
     }
 
+    private <T extends ActionRequest> T getLastRequest(Class<T> requestClass) {
+        final ActionRequest last = requests.get(requests.size() - 1);
+        assertThat(last, instanceOf(requestClass));
+        return requestClass.cast(last);
+    }
+
     private SearchHit[] buildHits(List<ApplicationPrivilegeDescriptor> sourcePrivileges) {
         final SearchHit[] hits = new SearchHit[sourcePrivileges.size()];
         for (int i = 0; i < hits.length; i++) {
@@ -1008,6 +820,51 @@ public class NativePrivilegeStoreTests extends ESTestCase {
         return hits;
     }
 
+    private static SearchResponse buildSearchResponse(SearchHit[] hits) {
+        return new SearchResponse(
+            new SearchResponseSections(
+                new SearchHits(hits, new TotalHits(hits.length, TotalHits.Relation.EQUAL_TO), 0f),
+                null,
+                null,
+                false,
+                false,
+                null,
+                1
+            ),
+            "_scrollId1",
+            1,
+            1,
+            0,
+            1,
+            null,
+            null
+        );
+    }
+
+    private void handleBulkRequest(int expectedCount, Predicate<DocWriteRequest<?>> isCreated) {
+        final BulkRequest bulkReq = getLastRequest(BulkRequest.class);
+        assertThat(bulkReq.requests(), hasSize(expectedCount));
+
+        final var uuid = UUIDs.randomBase64UUID(random());
+        final var items = new BulkItemResponse[expectedCount];
+        for (int i = 0; i < expectedCount; i++) {
+            final DocWriteRequest<?> itemReq = bulkReq.requests().get(i);
+            items[i] = BulkItemResponse.success(
+                i,
+                itemReq.opType(),
+                new IndexResponse(
+                    new ShardId(SecuritySystemIndices.SECURITY_MAIN_ALIAS, uuid, 0),
+                    itemReq.id(),
+                    1,
+                    1,
+                    1,
+                    isCreated.test(itemReq)
+                )
+            );
+        }
+        listener.get().onResponse(new BulkResponse(items, randomIntBetween(1, 999)));
+    }
+
     private void assertResult(
         List<ApplicationPrivilegeDescriptor> sourcePrivileges,
         PlainActionFuture<Collection<ApplicationPrivilegeDescriptor>> future
@@ -1017,6 +874,13 @@ public class NativePrivilegeStoreTests extends ESTestCase {
         assertThat(new HashSet<>(getPrivileges), equalTo(new HashSet<>(sourcePrivileges)));
     }
 
+    private void assertCacheCleared(String... applicationNames) {
+        final ClearPrivilegesCacheRequest clearCacheReq = getLastRequest(ClearPrivilegesCacheRequest.class);
+        assertThat(clearCacheReq.applicationNames(), arrayContainingInAnyOrder(applicationNames));
+        assertThat(clearCacheReq.clearRolesCache(), is(true));
+        listener.get().onResponse(new ClearPrivilegesCacheResponse(clusterService.getClusterName(), List.of(), List.of()));
+    }
+
     @SuppressWarnings("unchecked")
     private static <T> Consumer<T> anyConsumer() {
         return any(Consumer.class);