Allow access to restricted system indices for reserved system roles (#76845)

* Add system index patterns to TestRestrictedIndices

A missing piece in #74212 was system index patterns in the tests for the
ReservedRolesStore. Without these patterns, the tests did not accurately
check whether a role was incorrectly accessing a system index that was
not previously a restricted index.

This commit adds all of the current system index patterns to the test
class and adds restricted index access to the system roles that need it
for tests to pass.

* Preserve existing Kibana data telemetry privileges
* Test that data telemetry can't access security and async indices

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java

@@ -94,7 +94,7 @@ public class ReservedRolesStore implements BiConsumer<Set<String>, ActionListene
                             RoleDescriptor.IndicesPrivileges.builder()
                                 .indices("*").privileges("monitor").allowRestrictedIndices(true).build(),
                             RoleDescriptor.IndicesPrivileges.builder()
-                                .indices(".kibana*").privileges("read").build()
+                                .indices(".kibana*").privileges("read").allowRestrictedIndices(true).build()
                         },
                         null,
                         null,
@@ -199,6 +199,7 @@ public class ReservedRolesStore implements BiConsumer<Set<String>, ActionListene
                         new RoleDescriptor.IndicesPrivileges[] {
                                 RoleDescriptor.IndicesPrivileges.builder()
                                         .indices(".ml-anomalies*", ".ml-notifications*", ".ml-state*", ".ml-meta*", ".ml-stats-*")
+                                        .allowRestrictedIndices(true) // .ml-meta is a restricted index
                                         .privileges("view_index_metadata", "read").build(),
                                 RoleDescriptor.IndicesPrivileges.builder().indices(".ml-annotations*")
                                         .privileges("view_index_metadata", "read", "write").build()
@@ -258,12 +259,13 @@ public class ReservedRolesStore implements BiConsumer<Set<String>, ActionListene
                 .put("watcher_admin", new RoleDescriptor("watcher_admin", new String[] { "manage_watcher" },
                         new RoleDescriptor.IndicesPrivileges[] {
                                 RoleDescriptor.IndicesPrivileges.builder().indices(Watch.INDEX, TriggeredWatchStoreField.INDEX_NAME,
-                                        HistoryStoreField.INDEX_PREFIX + "*").privileges("read").build() },
+                                        HistoryStoreField.INDEX_PREFIX + "*").privileges("read").allowRestrictedIndices(true).build() },
                         null, MetadataUtils.DEFAULT_RESERVED_METADATA))
                 .put("watcher_user", new RoleDescriptor("watcher_user", new String[] { "monitor_watcher" },
                         new RoleDescriptor.IndicesPrivileges[] {
                                 RoleDescriptor.IndicesPrivileges.builder().indices(Watch.INDEX)
                                         .privileges("read")
+                                        .allowRestrictedIndices(true)
                                         .build(),
                                 RoleDescriptor.IndicesPrivileges.builder().indices(HistoryStoreField.INDEX_PREFIX + "*")
                                         .privileges("read")
@@ -272,6 +274,7 @@ public class ReservedRolesStore implements BiConsumer<Set<String>, ActionListene
                     new RoleDescriptor.IndicesPrivileges[] {
                         RoleDescriptor.IndicesPrivileges.builder().indices(".logstash*")
                                 .privileges("create", "delete", "index", "manage", "read")
+                                .allowRestrictedIndices(true)
                                 .build() },
                         null, MetadataUtils.DEFAULT_RESERVED_METADATA))
                 .put("rollup_user", new RoleDescriptor("rollup_user", new String[] { "monitor_rollup" },
@@ -369,8 +372,9 @@ public class ReservedRolesStore implements BiConsumer<Set<String>, ActionListene
                 "cancel_task"
             },
             new RoleDescriptor.IndicesPrivileges[] {
+                // System indices defined in KibanaPlugin
                 RoleDescriptor.IndicesPrivileges.builder()
-                    .indices(".kibana*", ".reporting-*").privileges("all").build(),
+                    .indices(".kibana*", ".reporting-*").privileges("all").allowRestrictedIndices(true).build(),
                 RoleDescriptor.IndicesPrivileges.builder()
                     .indices(".monitoring-*").privileges("read", "read_cross_cluster").build(),
                 RoleDescriptor.IndicesPrivileges.builder()
@@ -381,19 +385,20 @@ public class ReservedRolesStore implements BiConsumer<Set<String>, ActionListene
                     .privileges("read").build(),
                 RoleDescriptor.IndicesPrivileges.builder().indices(".ml-annotations*", ".ml-notifications*")
                     .privileges("read", "write").build(),
-                // APM agent configuration
+                // APM agent configuration - system index defined in KibanaPlugin
                 RoleDescriptor.IndicesPrivileges.builder()
-                    .indices(".apm-agent-configuration").privileges("all").build(),
-                // APM custom link index creation
+                    .indices(".apm-agent-configuration").privileges("all").allowRestrictedIndices(true).build(),
+                // APM custom link index creation - system index defined in KibanaPlugin
                 RoleDescriptor.IndicesPrivileges.builder()
-                    .indices(".apm-custom-link").privileges("all").build(),
+                    .indices(".apm-custom-link").privileges("all").allowRestrictedIndices(true).build(),
                 // APM telemetry queries APM indices in kibana task runner
                 RoleDescriptor.IndicesPrivileges.builder()
                     .indices("apm-*")
                     .privileges("read", "read_cross_cluster").build(),
-                // Data telemetry reads mappings, metadata and stats of indices
+                // Data telemetry reads mappings, metadata and stats of indices (excluding security and async search indices)
                 RoleDescriptor.IndicesPrivileges.builder()
-                    .indices("*")
+                    .indices("/@&~(\\.security.*)&~(\\.async-search.*)/")
+                    .allowRestrictedIndices(true)
                     .privileges("view_index_metadata", "monitor").build(),
                 // Endpoint diagnostic information. Kibana reads from these indices to send telemetry
                 RoleDescriptor.IndicesPrivileges.builder()
@@ -403,6 +408,7 @@ public class ReservedRolesStore implements BiConsumer<Set<String>, ActionListene
                 // Fleet Server indices. Kibana read and write to this indice to manage Elastic Agents
                 RoleDescriptor.IndicesPrivileges.builder()
                     .indices(".fleet*")
+                    .allowRestrictedIndices(true)
                     .privileges("all").build(),
                 // Legacy "Alerts as data" used in Security Solution.
                 // Kibana user creates these indices; reads / writes to them.

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java

@@ -532,7 +532,10 @@ public class ReservedRolesStoreTests extends ESTestCase {
 
 
         // Data telemetry reads mappings, metadata and stats of indices
-        Arrays.asList(randomAlphaOfLengthBetween(8, 24), "packetbeat-*", "logs-*").forEach((index) -> {
+        Arrays.asList(randomAlphaOfLengthBetween(8, 24), "packetbeat-*", "logs-*",
+            // check system indices other than .security* and .async-search*
+            ".watches", ".triggered-watches", ".tasks", ".enrich"
+        ).forEach((index) -> {
             logger.info("index name [{}]", index);
             assertThat(kibanaRole.indices().allowedIndicesMatcher(GetIndexAction.NAME).test(mockIndexAbstraction(index)), is(true));
             assertThat(kibanaRole.indices().allowedIndicesMatcher(GetMappingsAction.NAME).test(mockIndexAbstraction(index)), is(true));
@@ -550,6 +553,17 @@ public class ReservedRolesStoreTests extends ESTestCase {
             assertThat(kibanaRole.indices().allowedIndicesMatcher(READ_CROSS_CLUSTER_NAME).test(mockIndexAbstraction(index)), is(false));
         });
 
+        // Data telemetry does not have access to security and async search
+        RestrictedIndicesNames.RESTRICTED_NAMES.forEach((index) -> {
+            logger.info("index name [{}]", index);
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(GetIndexAction.NAME).test(mockIndexAbstraction(index)), is(false));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(GetMappingsAction.NAME).test(mockIndexAbstraction(index)), is(false));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(IndicesStatsAction.NAME).test(mockIndexAbstraction(index)), is(false));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher("indices:foo").test(mockIndexAbstraction(index)), is(false));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher("indices:bar").test(mockIndexAbstraction(index)), is(false));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(READ_CROSS_CLUSTER_NAME).test(mockIndexAbstraction(index)), is(false));
+        });
+
         // read-only datastream for Endpoint policy responses
         Arrays.asList("metrics-endpoint.policy-" + randomAlphaOfLength(randomIntBetween(0, 13))).forEach((index) -> {
             assertThat(kibanaRole.indices().allowedIndicesMatcher("indices:foo").test(mockIndexAbstraction(index)), is(false));

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/test/TestRestrictedIndices.java

@@ -23,6 +23,7 @@ import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -39,51 +40,140 @@ public class TestRestrictedIndices {
     public static final IndexNameExpressionResolver RESOLVER;
 
     static {
-        SystemIndices systemIndices = new SystemIndices(Map.of(
-            "security-mock",
+        Map<String, Feature> featureMap = new HashMap<>();
+        featureMap.put("security-mock",
             new Feature("security-mock", "fake security for test restricted indices", List.of(
-                SystemIndexDescriptor.builder()
-                    // This can't just be `.security-*` because that would overlap with the tokens index pattern
-                    .setIndexPattern(".security-[0-9]+")
-                    .setPrimaryIndex(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7)
-                    .setDescription("Contains Security configuration")
-                    .setMappings(mockMappings())
-                    .setSettings(Settings.EMPTY)
-                    .setAliasName(SECURITY_MAIN_ALIAS)
-                    .setIndexFormat(7)
-                    .setVersionMetaKey("version")
-                    .setOrigin(SECURITY_ORIGIN)
-                    .setThreadPools(ExecutorNames.CRITICAL_SYSTEM_INDEX_THREAD_POOLS)
-                    .build(),
-                SystemIndexDescriptor.builder()
-                    .setIndexPattern(".security-tokens-[0-9]+")
-                    .setPrimaryIndex(RestrictedIndicesNames.INTERNAL_SECURITY_TOKENS_INDEX_7)
-                    .setDescription("Contains auth token data")
-                    .setMappings(mockMappings())
-                    .setSettings(Settings.EMPTY)
-                    .setAliasName(SECURITY_TOKENS_ALIAS)
-                    .setIndexFormat(7)
-                    .setVersionMetaKey("version")
-                    .setOrigin(SECURITY_ORIGIN)
-                    .setThreadPools(ExecutorNames.CRITICAL_SYSTEM_INDEX_THREAD_POOLS)
-                    .build()
-            )),
-            "async-search-mock",
+                getMainSecurityDescriptor(),
+                getSecurityTokensDescriptor())));
+        featureMap.put("async-search-mock",
             new Feature("async search mock", "fake async search for restricted indices", List.of(
-                SystemIndexDescriptor.builder()
-                    .setIndexPattern(XPackPlugin.ASYNC_RESULTS_INDEX + "*")
-                    .setDescription("Async search results")
-                    .setPrimaryIndex(XPackPlugin.ASYNC_RESULTS_INDEX)
-                    .setMappings(mockMappings())
-                    .setSettings(Settings.EMPTY)
-                    .setVersionMetaKey("version")
-                    .setOrigin(ASYNC_SEARCH_ORIGIN)
-                    .build()
-            ))));
+                getAsyncSearchDescriptor())));
+        featureMap.put("kibana-mock",
+            new Feature("kibana-mock", "fake kibana for testing restricted indices", List.of(
+                getKibanaSavedObjectsDescriptor(),
+                getReportingIndexDescriptor(),
+                getApmAgentConfigDescriptor(),
+                getApmCustomLinkDescriptor())));
+
+        // From here, we have very minimal mock features that only supply system index patterns,
+        // not settings or mock mappings.
+        featureMap.put("enrich-mock",
+            new Feature("enrich-mock", "fake enrich for restricted indices tests", List.of(
+                new SystemIndexDescriptor(".enrich-*", "enrich pattern"))));
+        featureMap.put("fleet-mock",
+            new Feature("fleet-mock", "fake fleet for restricted indices tests", List.of(
+                new SystemIndexDescriptor(".fleet-actions~(-results*)", "fleet actions"),
+                new SystemIndexDescriptor(".fleet-agents*", "fleet agents"),
+                new SystemIndexDescriptor(".fleet-enrollment-api-keys*", "fleet enrollment"),
+                new SystemIndexDescriptor(".fleet-policies-[0-9]+", "fleet policies"),
+                new SystemIndexDescriptor(".fleet-policies-leader*", "fleet policies leader"),
+                new SystemIndexDescriptor(".fleet-servers*", "fleet servers"),
+                new SystemIndexDescriptor(".fleet-artifacts*", "fleet artifacts"))));
+        featureMap.put("ingest-geoip-mock",
+            new Feature("ingest-geoip-mock", "fake geoip for restricted indices tests", List.of(
+                new SystemIndexDescriptor(".geoip_databases", "geoip databases"))));
+        featureMap.put("logstash-mock",
+            new Feature("logstash-mock", "fake logstash for restricted indices tests", List.of(
+                new SystemIndexDescriptor(".logstash", "logstash"))));
+        featureMap.put("machine-learning-mock",
+            new Feature("machine-learning-mock", "fake machine learning for restricted indices tests", List.of(
+                new SystemIndexDescriptor(".ml-meta*", "machine learning meta"),
+                new SystemIndexDescriptor(".ml-config*", "machine learning config"),
+                new SystemIndexDescriptor(".ml-inference*", "machine learning inference"))));
+        featureMap.put("searchable-snapshots-mock",
+            new Feature("searchable-snapshots-mock", "fake searchable snapshots for restricted indices tests", List.of(
+                new SystemIndexDescriptor(".snapshot-blob-cache", "snapshot blob cache"))));
+        featureMap.put("transform-mock",
+            new Feature("transform-mock", "fake transform for restricted indices tests", List.of(
+                new SystemIndexDescriptor(".transform-internal-*", "transform internal"))));
+        featureMap.put("watcher-mock",
+            new Feature("watcher-mock", "fake watcher for restricted indices tests", List.of(
+                new SystemIndexDescriptor(".watches*", "watches"),
+                new SystemIndexDescriptor(".triggered-watches*", "triggered watches"))));
+
+        SystemIndices systemIndices = new SystemIndices(featureMap);
         RESTRICTED_INDICES_AUTOMATON = systemIndices.getSystemNameAutomaton();
         RESOLVER = TestIndexNameExpressionResolver.newInstance(systemIndices);
     }
 
+    private static SystemIndexDescriptor.Builder getInitializedDescriptorBuilder() {
+        return SystemIndexDescriptor.builder()
+            .setMappings(mockMappings())
+            .setSettings(Settings.EMPTY)
+            .setVersionMetaKey("version");
+    }
+
+    private static SystemIndexDescriptor getMainSecurityDescriptor() {
+        return getInitializedDescriptorBuilder()
+            // This can't just be `.security-*` because that would overlap with the tokens index pattern
+            .setIndexPattern(".security-[0-9]+")
+            .setPrimaryIndex(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7)
+            .setDescription("Contains Security configuration")
+            .setAliasName(SECURITY_MAIN_ALIAS)
+            .setIndexFormat(7)
+            .setOrigin(SECURITY_ORIGIN)
+            .setThreadPools(ExecutorNames.CRITICAL_SYSTEM_INDEX_THREAD_POOLS)
+            .build();
+    }
+
+    private static SystemIndexDescriptor getSecurityTokensDescriptor() {
+        return getInitializedDescriptorBuilder()
+            .setIndexPattern(".security-tokens-[0-9]+")
+            .setPrimaryIndex(RestrictedIndicesNames.INTERNAL_SECURITY_TOKENS_INDEX_7)
+            .setDescription("Contains auth token data")
+            .setAliasName(SECURITY_TOKENS_ALIAS)
+            .setIndexFormat(7)
+            .setOrigin(SECURITY_ORIGIN)
+            .setThreadPools(ExecutorNames.CRITICAL_SYSTEM_INDEX_THREAD_POOLS)
+            .build();
+    }
+
+    private static SystemIndexDescriptor getAsyncSearchDescriptor() {
+        return getInitializedDescriptorBuilder()
+            .setIndexPattern(XPackPlugin.ASYNC_RESULTS_INDEX + "*")
+            .setDescription("Async search results")
+            .setPrimaryIndex(XPackPlugin.ASYNC_RESULTS_INDEX)
+            .setOrigin(ASYNC_SEARCH_ORIGIN)
+            .build();
+    }
+
+    private static SystemIndexDescriptor getKibanaSavedObjectsDescriptor() {
+        return SystemIndexDescriptor.builder()
+            .setIndexPattern(".kibana_*")
+            .setDescription("Kibana saved objects system index")
+            .setAliasName(".kibana")
+            .setType(SystemIndexDescriptor.Type.EXTERNAL_UNMANAGED)
+            .setAllowedElasticProductOrigins( List.of("kibana"))
+            .build();
+    }
+
+    private static SystemIndexDescriptor getReportingIndexDescriptor() {
+        return SystemIndexDescriptor.builder()
+            .setIndexPattern(".reporting-*")
+            .setDescription("system index for reporting")
+            .setType(SystemIndexDescriptor.Type.EXTERNAL_UNMANAGED)
+            .setAllowedElasticProductOrigins(List.of("kibana"))
+            .build();
+    }
+
+    private static SystemIndexDescriptor getApmAgentConfigDescriptor() {
+        return SystemIndexDescriptor.builder()
+            .setIndexPattern(".apm-agent-configuration")
+            .setDescription("system index for APM agent configuration")
+            .setType(SystemIndexDescriptor.Type.EXTERNAL_UNMANAGED)
+            .setAllowedElasticProductOrigins(List.of("kibana"))
+            .build();
+    }
+
+    private static SystemIndexDescriptor getApmCustomLinkDescriptor() {
+        return SystemIndexDescriptor.builder()
+            .setIndexPattern(".apm-custom-link")
+            .setDescription("system index for APM custom links")
+            .setType(SystemIndexDescriptor.Type.EXTERNAL_UNMANAGED)
+            .setAllowedElasticProductOrigins(List.of("kibana"))
+            .build();
+    }
+
     private TestRestrictedIndices() {}
 
     private static XContentBuilder mockMappings() {