Permissions required for stateful agentless integrations (#118644) (#119973)

Closes elastic/security-team#11102
Closes elastic/security-team#11104

This allows agentless integrations (via elastic/beats#41446, elastic/kibana#203810) to write to agentless-* indices. Each index is created on-demand by the filebeat client and kibana conditionally extends the API key permissions to allow writing to the index.

(cherry picked from commit 3c184b912c21282419d9ac4b5b71d04215e2b30e)

# Conflicts:
#	docs/reference/rest-api/security/get-service-accounts.asciidoc
#	x-pack/plugin/security/qa/service-account/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/service/ServiceAccountIT.java
#	x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/service/ElasticServiceAccounts.java

docs/reference/rest-api/security/get-service-accounts.asciidoc

@@ -244,6 +244,21 @@ GET /_security/service/elastic/fleet-server
             "auto_configure"
           ],
           "allow_restricted_indices": false
+        },
+        {
+          "names": [
+            "agentless-*",
+          ],
+          "privileges": [
+            "read",
+            "write",
+            "monitor",
+            "create_index",
+            "auto_configure",
+            "maintenance",
+            "view_index_metadata"
+          ],
+          "allow_restricted_indices": false
         }
       ],
       "applications": [

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -808,6 +808,7 @@ public abstract class ESRestTestCase extends ESTestCase {
             "profiling-60-days",
             "profiling-60-days@lifecycle",
             "synthetics",
+            "agentless",
             "synthetics@lifecycle",
             "traces@lifecycle",
             "7-days-default",
@@ -2260,6 +2261,7 @@ public abstract class ESRestTestCase extends ESTestCase {
             case "metrics-tsdb-settings":
             case "metrics-mappings":
             case "synthetics":
+            case "agentless":
             case "synthetics-settings":
             case "synthetics-mappings":
             case ".snapshot-blob-cache":

x-pack/plugin/core/template-resources/src/main/resources/agentless@mappings.json

@@ -0,0 +1,22 @@
+{
+  "template": {
+    "mappings": {
+      "properties": {
+        "v": {
+          "type": "object",
+          "enabled": false
+        },
+        "updated_at": {
+          "type": "date",
+          "format": "strict_date_optional_time||epoch_millis"
+        }
+      }
+    }
+  },
+  "_meta": {
+    "description": "default mappings for the agentless index template installed by x-pack",
+    "managed": true
+  },
+  "version": ${xpack.stack.template.version},
+  "deprecated": ${xpack.stack.template.deprecated}
+}

x-pack/plugin/core/template-resources/src/main/resources/agentless@settings.json

@@ -0,0 +1,15 @@
+{
+  "template": {
+    "settings": {
+      "index": {
+        "hidden": true
+      }
+    }
+  },
+  "_meta": {
+    "description": "default settings for the agentless index template installed by x-pack",
+    "managed": true
+  },
+  "version": ${xpack.stack.template.version},
+  "deprecated": ${xpack.stack.template.deprecated}
+}

x-pack/plugin/core/template-resources/src/main/resources/agentless@template.json

@@ -0,0 +1,15 @@
+{
+  "index_patterns": ["agentless-*-*"],
+  "priority": 100,
+  "composed_of": [
+    "agentless@mappings",
+    "agentless@settings"
+  ],
+  "allow_auto_create": true,
+  "_meta": {
+    "description": "default agentless template installed by x-pack",
+    "managed": true
+  },
+  "version": ${xpack.stack.template.version},
+  "deprecated": ${xpack.stack.template.deprecated}
+}

x-pack/plugin/security/qa/service-account/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/service/ServiceAccountIT.java

@@ -284,6 +284,21 @@ public class ServiceAccountIT extends ESRestTestCase {
                     "auto_configure"
                   ],
                   "allow_restricted_indices": false
+                },
+                {
+                  "names": [
+                    "agentless-*"
+                  ],
+                  "privileges": [
+                    "read",
+                    "write",
+                    "monitor",
+                    "create_index",
+                    "auto_configure",
+                    "maintenance",
+                    "view_index_metadata"
+                  ],
+                  "allow_restricted_indices": false
                 }
               ],
               "applications": [        {

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/service/ElasticServiceAccounts.java

@@ -156,7 +156,13 @@ final class ElasticServiceAccounts {
                     // Fleet Server needs "read" privilege to be able to retrieve multi-agent docs
                     .privileges("read", "write", "create_index", "auto_configure")
                     .allowRestrictedIndices(false)
-                    .build() },
+                    .build(),
+                // Custom permissions required for stateful agentless integrations
+                RoleDescriptor.IndicesPrivileges.builder()
+                    .indices("agentless-*")
+                    .privileges("read", "write", "monitor", "create_index", "auto_configure", "maintenance", "view_index_metadata")
+                    .allowRestrictedIndices(false)
+                    .build(), },
             new RoleDescriptor.ApplicationResourcePrivileges[] {
                 RoleDescriptor.ApplicationResourcePrivileges.builder()
                     .application("kibana-*")

x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java

@@ -111,6 +111,13 @@ public class StackTemplateRegistry extends IndexTemplateRegistry {
     public static final String SYNTHETICS_ILM_POLICY_NAME = "synthetics@lifecycle";
     public static final String SYNTHETICS_INDEX_TEMPLATE_NAME = "synthetics";
 
+    //////////////////////////////////////////////////////////
+    // Agentless components (for matching agentless-*-* indices)
+    //////////////////////////////////////////////////////////
+    public static final String AGENTLESS_MAPPINGS_COMPONENT_TEMPLATE_NAME = "agentless@mappings";
+    public static final String AGENTLESS_SETTINGS_COMPONENT_TEMPLATE_NAME = "agentless@settings";
+    public static final String AGENTLESS_INDEX_TEMPLATE_NAME = "agentless";
+
     ///////////////////////////////////
     // Kibana reporting template
     ///////////////////////////////////
@@ -212,6 +219,20 @@ public class StackTemplateRegistry extends IndexTemplateRegistry {
                 TEMPLATE_VERSION_VARIABLE,
                 ADDITIONAL_TEMPLATE_VARIABLES
             ),
+            new IndexTemplateConfig(
+                AGENTLESS_MAPPINGS_COMPONENT_TEMPLATE_NAME,
+                "/agentless@mappings.json",
+                REGISTRY_VERSION,
+                TEMPLATE_VERSION_VARIABLE,
+                ADDITIONAL_TEMPLATE_VARIABLES
+            ),
+            new IndexTemplateConfig(
+                AGENTLESS_SETTINGS_COMPONENT_TEMPLATE_NAME,
+                "/agentless@settings.json",
+                REGISTRY_VERSION,
+                TEMPLATE_VERSION_VARIABLE,
+                ADDITIONAL_TEMPLATE_VARIABLES
+            ),
             new IndexTemplateConfig(
                 KIBANA_REPORTING_COMPONENT_TEMPLATE_NAME,
                 "/kibana-reporting@settings.json",
@@ -300,6 +321,13 @@ public class StackTemplateRegistry extends IndexTemplateRegistry {
             TEMPLATE_VERSION_VARIABLE,
             ADDITIONAL_TEMPLATE_VARIABLES
         ),
+        new IndexTemplateConfig(
+            AGENTLESS_INDEX_TEMPLATE_NAME,
+            "/agentless@template.json",
+            REGISTRY_VERSION,
+            TEMPLATE_VERSION_VARIABLE,
+            ADDITIONAL_TEMPLATE_VARIABLES
+        ),
         new IndexTemplateConfig(
             KIBANA_REPORTING_INDEX_TEMPLATE_NAME,
             "/kibana-reporting@template.json",

x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java

@@ -430,6 +430,8 @@ public class StackTemplateRegistryTests extends ESTestCase {
         versions.put(StackTemplateRegistry.METRICS_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
         versions.put(StackTemplateRegistry.SYNTHETICS_SETTINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
         versions.put(StackTemplateRegistry.SYNTHETICS_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
+        versions.put(StackTemplateRegistry.AGENTLESS_SETTINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
+        versions.put(StackTemplateRegistry.AGENTLESS_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
         versions.put(StackTemplateRegistry.KIBANA_REPORTING_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
         versions.put(StackTemplateRegistry.TRACES_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
         versions.put(StackTemplateRegistry.TRACES_SETTINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
@@ -488,6 +490,14 @@ public class StackTemplateRegistryTests extends ESTestCase {
             StackTemplateRegistry.SYNTHETICS_MAPPINGS_COMPONENT_TEMPLATE_NAME,
             StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000)
         );
+        versions.put(
+            StackTemplateRegistry.AGENTLESS_SETTINGS_COMPONENT_TEMPLATE_NAME,
+            StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000)
+        );
+        versions.put(
+            StackTemplateRegistry.AGENTLESS_MAPPINGS_COMPONENT_TEMPLATE_NAME,
+            StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000)
+        );
         versions.put(
             StackTemplateRegistry.KIBANA_REPORTING_COMPONENT_TEMPLATE_NAME,
             StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000)