[Fleet] Add `.fleet-secrets` system index (#95625)

* add .secrets index

* spotless

* remove system.out

* fix descriptor test

* t push fix test

* fix test

* change index description

* add missing .

* restore docs

* Update docs/changelog/95625.yaml

* renam .secrets to .fleet-secrets

* checkstyle + add comment

* spotless

* change order of privs

* Update docs/changelog/95625.yaml

* Update 95625.yaml

* Update FleetSystemIndicesIT.java

* Update Fleet.java

---------

Co-authored-by: Julia Bardi <90178898+juliaElastic@users.noreply.github.com>

docs/changelog/95625.yaml

@@ -0,0 +1,6 @@
+pr: 95625
+summary: "[Fleet] Add `.fleet-secrets` system index"
+area: Infra/Plugins
+type: feature
+issues:
+ - 95143

x-pack/docs/en/rest-api/security/get-service-accounts.asciidoc

@@ -105,6 +105,15 @@ GET /_security/service/elastic/fleet-server
           ],
           "allow_restricted_indices": false
         },
+        {
+          "names": [
+            ".fleet-secrets*"
+          ],
+          "privileges": [
+            "read",
+          ],
+          "allow_restricted_indices": true
+        },
         {
           "names": [
             ".fleet-*"

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java

@@ -764,6 +764,13 @@ public class ReservedRolesStore implements BiConsumer<Set<String>, ActionListene
                 RoleDescriptor.IndicesPrivileges.builder().indices("*").privileges("view_index_metadata", "monitor").build(),
                 // Endpoint diagnostic information. Kibana reads from these indices to send telemetry
                 RoleDescriptor.IndicesPrivileges.builder().indices(".logs-endpoint.diagnostic.collection-*").privileges("read").build(),
+                // Fleet secrets, Kibana can only write ot this index.
+                // This definition must come before .fleet* below.
+                RoleDescriptor.IndicesPrivileges.builder()
+                    .indices(".fleet-secrets*")
+                    .privileges("write", "create_index")
+                    .allowRestrictedIndices(true)
+                    .build(),
                 // Fleet Server indices. Kibana create this indice before Fleet Server use them.
                 // Fleet Server indices. Kibana read and write to this indice to manage Elastic Agents
                 RoleDescriptor.IndicesPrivileges.builder().indices(".fleet*").allowRestrictedIndices(true).privileges("all").build(),

x-pack/plugin/core/src/main/resources/fleet-secrets.json

@@ -0,0 +1,19 @@
+{
+  "settings": {
+    "auto_expand_replicas": "0-1"
+  },
+  "mappings": {
+    "_doc" : {
+      "dynamic": false,
+      "_meta": {
+        "version": "${fleet.version}"
+      },
+      "properties": {
+        "value": {
+          "type": "keyword",
+          "index": false
+        }
+      }
+    }
+  }
+}

x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java

@@ -151,6 +151,22 @@ public class FleetSystemIndicesIT extends ESRestTestCase {
         assertThat(responseBody, containsString("coordinator_idx"));
     }
 
+    public void testCreationOfFleetSecrets() throws Exception {
+        Request request = new Request("PUT", ".fleet-secrets");
+        Response response = client().performRequest(request);
+        assertEquals(200, response.getStatusLine().getStatusCode());
+
+        request = new Request("GET", ".fleet-secrets/_mapping");
+        response = client().performRequest(request);
+        String responseBody = EntityUtils.toString(response.getEntity());
+        assertThat(responseBody, containsString("value"));
+
+        request = new Request("GET", ".fleet-secrets-7/_mapping");
+        response = client().performRequest(request);
+        responseBody = EntityUtils.toString(response.getEntity());
+        assertThat(responseBody, containsString("value"));
+    }
+
     public void testCreationOfFleetPoliciesLeader() throws Exception {
         Request request = new Request("PUT", ".fleet-policies-leader");
         Response response = client().performRequest(request);

x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java

@@ -114,6 +114,7 @@ public class Fleet extends Plugin implements SystemIndexPlugin {
             fleetActionsSystemIndexDescriptor(),
             fleetAgentsSystemIndexDescriptor(),
             fleetEnrollmentApiKeysSystemIndexDescriptor(),
+            fleetSecretsSystemIndexDescriptor(),
             fleetPoliciesSystemIndexDescriptor(),
             fleetPoliciesLeaderSystemIndexDescriptor(),
             fleetServersSystemIndexDescriptors(),
@@ -190,6 +191,23 @@ public class Fleet extends Plugin implements SystemIndexPlugin {
             .build();
     }
 
+    private SystemIndexDescriptor fleetSecretsSystemIndexDescriptor() {
+        PutIndexTemplateRequest request = new PutIndexTemplateRequest();
+        request.source(loadTemplateSource("/fleet-secrets.json"), XContentType.JSON);
+        return SystemIndexDescriptor.builder()
+            .setType(Type.EXTERNAL_MANAGED)
+            .setAllowedElasticProductOrigins(ALLOWED_PRODUCTS)
+            .setOrigin(FLEET_ORIGIN)
+            .setVersionMetaKey(VERSION_KEY)
+            .setMappings(request.mappings())
+            .setSettings(request.settings())
+            .setPrimaryIndex(".fleet-secrets-" + CURRENT_INDEX_VERSION)
+            .setIndexPattern(".fleet-secrets*")
+            .setAliasName(".fleet-secrets")
+            .setDescription("Secret values managed by Fleet")
+            .build();
+    }
+
     private SystemIndexDescriptor fleetPoliciesSystemIndexDescriptor() {
         PutIndexTemplateRequest request = new PutIndexTemplateRequest();
         request.source(loadTemplateSource("/fleet-policies.json"), XContentType.JSON);

x-pack/plugin/fleet/src/test/java/org/elasticsearch/xpack/fleet/FleetTests.java

@@ -35,7 +35,8 @@ public class FleetTests extends ESTestCase {
                 ".fleet-actions~(-results*)",
                 ".fleet-policies-leader*",
                 ".fleet-enrollment-api-keys*",
-                ".fleet-artifacts*"
+                ".fleet-artifacts*",
+                ".fleet-secrets*"
             )
         );
 

x-pack/plugin/security/qa/service-account/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/service/ServiceAccountIT.java

@@ -124,6 +124,15 @@ public class ServiceAccountIT extends ESRestTestCase {
                   ],
                   "allow_restricted_indices": false
                 },
+                {
+                  "names": [
+                    ".fleet-secrets*"
+                  ],
+                  "privileges": [
+                    "read"
+                  ],
+                  "allow_restricted_indices":true
+                },
                 {
                   "names": [
                     ".fleet-*"

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/service/ElasticServiceAccounts.java

@@ -83,6 +83,11 @@ final class ElasticServiceAccounts {
                     .indices("traces-apm.sampled-*")
                     .privileges("read", "monitor", "maintenance")
                     .build(),
+                RoleDescriptor.IndicesPrivileges.builder()
+                    .indices(".fleet-secrets*")
+                    .privileges("read")
+                    .allowRestrictedIndices(true)
+                    .build(),
                 RoleDescriptor.IndicesPrivileges.builder()
                     .indices(".fleet-*")
                     // Fleet Server needs "maintenance" privilege to be able to perform operations with "refresh"