[Fleet] Add files and files data index templates and ILM policies (#91413)

* add files system index templates and ilm policies

* add tests

* fix templates

* fix index templates

* fix index pattern

* fix import order

* mark .fleet index templates as xpack

* implement suggestions made by @andreidan

* spotlessApply

* update SearchUsageStatsIT

* add collectors import

* add missing docs

* spotless

* Supress correct warnings

* spotless

* fix warnings

* line length

* do not resuse req options

* add logging

* move to new var

* fix assertions

* fix fileData assertion

* fix version templating

* Update docs/changelog/91413.yaml

* remove docs not needed

* delete files after 7d

docs/changelog/91413.yaml

@@ -0,0 +1,5 @@
+pr: 91413
+summary: "[Fleet] Add files and files data index templates and ILM policies"
+area: Infra/Plugins
+type: feature
+issues: []

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -602,6 +602,8 @@ public abstract class ESRestTestCase extends ESTestCase {
             "180-days-default",
             "365-days-default",
             ".fleet-actions-results-ilm-policy",
+            ".fleet-file-data-ilm-policy",
+            ".fleet-files-ilm-policy",
             ".deprecation-indexing-ilm-policy",
             ".monitoring-8-ilm-policy"
         );
@@ -1774,6 +1776,9 @@ public abstract class ESRestTestCase extends ESTestCase {
         if (name.startsWith(".deprecation-")) {
             return true;
         }
+        if (name.startsWith(".fleet-")) {
+            return true;
+        }
         switch (name) {
             case ".watches":
             case "security_audit_log":

x-pack/plugin/core/src/main/resources/fleet-file-data-ilm-policy.json

@@ -0,0 +1,25 @@
+{
+  "phases": {
+    "hot": {
+      "min_age": "0ms",
+      "actions": {
+        "rollover": {
+          "max_size": "10gb",
+          "max_age": "7d"
+        }
+      }
+    },
+    "delete": {
+      "min_age": "7d",
+      "actions": {
+        "delete": {
+          "delete_searchable_snapshot": true
+        }
+      }
+    }
+  },
+  "_meta": {
+    "description": "policy for fleet uploaded files",
+    "managed": true
+  }
+}

x-pack/plugin/core/src/main/resources/fleet-file-data.json

@@ -0,0 +1,38 @@
+{
+  "index_patterns": [
+    ".fleet-file-data-*-*"
+  ],
+  "priority": 200,
+  "composed_of": [],
+  "_meta": {
+    "description": "fleet file data index template",
+    "managed": true
+  },
+  "template" : {
+    "settings": {
+      "index.lifecycle.name": ".fleet-file-data-ilm-policy",
+      "auto_expand_replicas": "0-1"
+    },
+    "mappings": {
+      "_doc": {
+        "_meta": {
+          "version": "${xpack.fleet.template.version}"
+        },
+        "properties": {
+          "data": {
+            "type": "binary"
+          },
+          "bid": {
+            "type": "keyword",
+            "index": false
+          },
+          "last": {
+            "type": "boolean",
+            "index": false
+          }
+        }
+      }
+    }
+  },
+  "version": ${xpack.fleet.template.version}
+}

x-pack/plugin/core/src/main/resources/fleet-files-ilm-policy.json

@@ -0,0 +1,25 @@
+{
+  "phases": {
+    "hot": {
+      "min_age": "0ms",
+      "actions": {
+        "rollover": {
+          "max_size": "10gb",
+          "max_age": "30d"
+        }
+      }
+    },
+    "delete": {
+      "min_age": "90d",
+      "actions": {
+        "delete": {
+          "delete_searchable_snapshot": true
+        }
+      }
+    }
+  },
+  "_meta": {
+    "description": "policy for fleet uploaded file metadata",
+    "managed": true
+  }
+}

x-pack/plugin/core/src/main/resources/fleet-files.json

@@ -0,0 +1,53 @@
+{
+  "index_patterns": [
+    ".fleet-files-*-*"
+  ],
+  "priority": 200,
+  "composed_of": [],
+  "_meta": {
+    "description": "fleet files index template",
+    "managed": true
+  },
+  "template": {
+    "settings": {
+      "index.lifecycle.name": ".fleet-files-ilm-policy",
+      "index.auto_expand_replicas": "0-1"
+    },
+    "mappings": {
+      "_doc": {
+        "_meta": {
+          "version": "${xpack.fleet.template.version}"
+        },
+        "dynamic": false,
+        "properties": {
+          "agent_id": {
+            "type": "keyword"
+          },
+          "action_id": {
+            "type": "keyword"
+          },
+          "source": {
+            "type": "keyword"
+          },
+          "file": {
+            "properties": {
+              "Status": {
+                "type": "keyword"
+              },
+              "ChunkSize": {
+                "type": "integer"
+              },
+              "Compression": {
+                "type": "keyword"
+              },
+              "name": {
+                "type": "keyword"
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "version": ${xpack.fleet.template.version}
+}

x-pack/plugin/fleet/build.gradle

@@ -21,6 +21,7 @@ dependencies {
   testImplementation(testArtifact(project(xpackModule('core'))))
   javaRestTestImplementation(project(path: xpackModule('core')))
   javaRestTestImplementation(testArtifact(project(xpackModule('core'))))
+  compileOnly project(path: xpackModule('ilm'))
 }
 
 testClusters.configureEach {

x-pack/plugin/fleet/src/internalClusterTest/java/org/elasticsearch/xpack/fleet/action/GetGlobalCheckpointsActionIT.java

@@ -23,7 +23,9 @@ import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
 import org.elasticsearch.xpack.fleet.Fleet;
+import org.elasticsearch.xpack.ilm.IndexLifecycle;
 
 import java.util.Arrays;
 import java.util.Collection;
@@ -43,7 +45,7 @@ public class GetGlobalCheckpointsActionIT extends ESIntegTestCase {
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Stream.of(Fleet.class).collect(Collectors.toList());
+        return Stream.of(Fleet.class, LocalStateCompositeXPackPlugin.class, IndexLifecycle.class).collect(Collectors.toList());
     }
 
     public void testGetGlobalCheckpoints() throws Exception {

x-pack/plugin/fleet/src/internalClusterTest/java/org/elasticsearch/xpack/fleet/action/SearchUsageStatsIT.java

@@ -16,11 +16,14 @@ import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
 import org.elasticsearch.xpack.fleet.Fleet;
+import org.elasticsearch.xpack.ilm.IndexLifecycle;
 
 import java.io.IOException;
 import java.util.Collection;
-import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 1)
 public class SearchUsageStatsIT extends ESIntegTestCase {
@@ -32,7 +35,7 @@ public class SearchUsageStatsIT extends ESIntegTestCase {
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return List.of(Fleet.class);
+        return Stream.of(Fleet.class, LocalStateCompositeXPackPlugin.class, IndexLifecycle.class).collect(Collectors.toList());
     }
 
     public void testSearchUsageStats() throws IOException {

x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java

@@ -9,6 +9,7 @@ package org.elasticsearch.xpack.fleet;
 
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -17,10 +18,12 @@ import org.elasticsearch.test.SecuritySettingsSourceField;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.xcontent.XContentType;
 
+import java.util.List;
 import java.util.Map;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
 
 public class FleetSystemIndicesIT extends ESRestTestCase {
 
@@ -37,6 +40,22 @@ public class FleetSystemIndicesIT extends ESRestTestCase {
             .build();
     }
 
+    private void expectSystemIndexWarning(Request request, String indexName) {
+        String warningMsg = "index name ["
+            + indexName
+            + "] starts with a dot '.', in the next major version, "
+            + "index names starting with a dot are reserved for hidden indices and system indices";
+
+        List<String> expectedWarnings = List.of(warningMsg);
+
+        logger.info("expecting warnings: " + expectedWarnings.toString());
+        RequestOptions consumeSystemIndicesWarningsOptions = RequestOptions.DEFAULT.toBuilder()
+            .setWarningsHandler(warnings -> expectedWarnings.equals(warnings) == false)
+            .build();
+
+        request.setOptions(consumeSystemIndicesWarningsOptions);
+    }
+
     public void testSearchWithoutIndexCreatedIsAllowed() throws Exception {
         Request request = new Request("GET", ".fleet-agents/_search");
         request.setJsonEntity("{ \"query\": { \"match_all\": {} } }");
@@ -77,6 +96,33 @@ public class FleetSystemIndicesIT extends ESRestTestCase {
         assertThat(responseBody, containsString("action_id"));
     }
 
+    public void testCreationOfFleetFiles() throws Exception {
+        Request request = new Request("PUT", ".fleet-files-agent-00001");
+        expectSystemIndexWarning(request, ".fleet-files-agent-00001");
+        Response response = client().performRequest(request);
+        assertEquals(200, response.getStatusLine().getStatusCode());
+
+        request = new Request("GET", ".fleet-files-agent-00001/_mapping");
+        response = client().performRequest(request);
+        String responseBody = EntityUtils.toString(response.getEntity());
+        assertThat(responseBody, not(containsString("xpack.fleet.template.version"))); // assert templating worked
+        assertThat(responseBody, containsString("action_id"));
+    }
+
+    public void testCreationOfFleetFileData() throws Exception {
+        Request request = new Request("PUT", ".fleet-file-data-agent-00001");
+        expectSystemIndexWarning(request, ".fleet-file-data-agent-00001");
+        Response response = client().performRequest(request);
+        assertEquals(200, response.getStatusLine().getStatusCode());
+
+        request = new Request("GET", ".fleet-file-data-agent-00001/_mapping");
+        response = client().performRequest(request);
+        String responseBody = EntityUtils.toString(response.getEntity());
+        assertThat(responseBody, not(containsString("xpack.fleet.template.version"))); // assert templating worked
+        assertThat(responseBody, containsString("data"));
+        assertThat(responseBody, containsString("bid"));
+    }
+
     public void testCreationOfFleetArtifacts() throws Exception {
         Request request = new Request("PUT", ".fleet-artifacts");
         Response response = client().performRequest(request);
@@ -165,7 +211,7 @@ public class FleetSystemIndicesIT extends ESRestTestCase {
     }
 
     @SuppressWarnings("unchecked")
-    public void verifyILMPolicyExists() throws Exception {
+    public void verifyActionsILMPolicyExists() throws Exception {
         assertBusy(() -> {
             Request request = new Request("GET", "_ilm/policy/.fleet-actions-results-ilm-policy");
             Response response = client().performRequest(request);
@@ -178,4 +224,34 @@ public class FleetSystemIndicesIT extends ESRestTestCase {
             assertThat(policyMap.size(), equalTo(2));
         });
     }
+
+    @SuppressWarnings("unchecked")
+    public void verifyFilesILMPolicyExists() throws Exception {
+        assertBusy(() -> {
+            Request request = new Request("GET", "_ilm/policy/.fleet-files-ilm-policy");
+            Response response = client().performRequest(request);
+            assertEquals(200, response.getStatusLine().getStatusCode());
+            final String responseJson = EntityUtils.toString(response.getEntity());
+            Map<String, Object> responseMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), responseJson, false);
+            assertNotNull(responseMap.get(".fleet-files-ilm-policy"));
+            Map<String, Object> policyMap = (Map<String, Object>) responseMap.get(".fleet-files-ilm-policy");
+            assertNotNull(policyMap);
+            assertThat(policyMap.size(), equalTo(2));
+        });
+    }
+
+    @SuppressWarnings("unchecked")
+    public void verifyFileDataILMPolicyExists() throws Exception {
+        assertBusy(() -> {
+            Request request = new Request("GET", "_ilm/policy/.fleet-file-data-ilm-policy");
+            Response response = client().performRequest(request);
+            assertEquals(200, response.getStatusLine().getStatusCode());
+            final String responseJson = EntityUtils.toString(response.getEntity());
+            Map<String, Object> responseMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), responseJson, false);
+            assertNotNull(responseMap.get(".fleet-file-data-ilm-policy"));
+            Map<String, Object> policyMap = (Map<String, Object>) responseMap.get(".fleet-file-data-ilm-policy");
+            assertNotNull(policyMap);
+            assertThat(policyMap.size(), equalTo(2));
+        });
+    }
 }

x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/FleetTemplateRegistry.java

@@ -8,25 +8,43 @@
 package org.elasticsearch.xpack.fleet;
 
 import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.ilm.LifecyclePolicy;
+import org.elasticsearch.xpack.core.template.IndexTemplateConfig;
 import org.elasticsearch.xpack.core.template.IndexTemplateRegistry;
 import org.elasticsearch.xpack.core.template.LifecyclePolicyConfig;
 
 import java.util.List;
+import java.util.Map;
 
 public class FleetTemplateRegistry extends IndexTemplateRegistry {
 
+    public static final int INDEX_TEMPLATE_VERSION = 1;
+
+    public static final String TEMPLATE_VERSION_VARIABLE = "xpack.fleet.template.version";
+
     private static final List<LifecyclePolicy> LIFECYCLE_POLICIES = List.of(
         new LifecyclePolicyConfig(".fleet-actions-results-ilm-policy", "/fleet-actions-results-ilm-policy.json").load(
             LifecyclePolicyConfig.DEFAULT_X_CONTENT_REGISTRY
+        ),
+        new LifecyclePolicyConfig(".fleet-file-data-ilm-policy", "/fleet-file-data-ilm-policy.json").load(
+            LifecyclePolicyConfig.DEFAULT_X_CONTENT_REGISTRY
+        ),
+        new LifecyclePolicyConfig(".fleet-files-ilm-policy", "/fleet-files-ilm-policy.json").load(
+            LifecyclePolicyConfig.DEFAULT_X_CONTENT_REGISTRY
         )
     );
 
+    public static final Map<String, ComposableIndexTemplate> COMPOSABLE_INDEX_TEMPLATE_CONFIGS = parseComposableTemplates(
+        new IndexTemplateConfig(".fleet-files", "/fleet-files.json", INDEX_TEMPLATE_VERSION, TEMPLATE_VERSION_VARIABLE),
+        new IndexTemplateConfig(".fleet-file-data", "/fleet-file-data.json", INDEX_TEMPLATE_VERSION, TEMPLATE_VERSION_VARIABLE)
+    );
+
     public FleetTemplateRegistry(
         Settings nodeSettings,
         ClusterService clusterService,
@@ -46,4 +64,9 @@ public class FleetTemplateRegistry extends IndexTemplateRegistry {
     protected List<LifecyclePolicy> getPolicyConfigs() {
         return LIFECYCLE_POLICIES;
     }
+
+    @Override
+    protected Map<String, ComposableIndexTemplate> getComposableTemplateConfigs() {
+        return COMPOSABLE_INDEX_TEMPLATE_CONFIGS;
+    }
 }

x-pack/plugin/ilm/src/main/java/module-info.java

@@ -13,7 +13,7 @@ module org.elasticsearch.ilm {
     requires org.apache.logging.log4j;
 
     exports org.elasticsearch.xpack.ilm.action to org.elasticsearch.server;
-    exports org.elasticsearch.xpack.ilm to org.elasticsearch.server;
+    exports org.elasticsearch.xpack.ilm;
     exports org.elasticsearch.xpack.slm.action to org.elasticsearch.server;
     exports org.elasticsearch.xpack.slm to org.elasticsearch.server;