Add non-indexed fields to ecs templates (#106714)

* add non-indexed fields to ecs templates

* update

* Update docs/changelog/106714.yaml

* Adjusting ECS tests to verify proper index and doc_values settings

* review comment

---------

Co-authored-by: eyalkoren <41850454+eyalkoren@users.noreply.github.com>

docs/changelog/106714.yaml

@@ -0,0 +1,5 @@
+pr: 106714
+summary: Add non-indexed fields to ecs templates
+area: Data streams
+type: bug
+issues: []

x-pack/plugin/core/template-resources/src/main/resources/ecs@mappings.json

@@ -23,6 +23,30 @@
             "unmatch_mapping_type": "object"
           }
         },
+        {
+          "ecs_non_indexed_keyword": {
+            "mapping": {
+              "type": "keyword",
+              "index": false,
+              "doc_values": false
+            },
+            "path_match": [
+              "event.original"
+            ]
+          }
+        },
+        {
+          "ecs_non_indexed_long": {
+            "mapping": {
+              "type": "long",
+              "index": false,
+              "doc_values": false
+            },
+            "path_match": [
+              "*.x509.public_key_exponent"
+            ]
+          }
+        },
         {
           "ecs_ip": {
             "mapping": {

x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java

@@ -191,6 +191,11 @@ public class EcsDynamicTemplatesIT extends ESRestTestCase {
         verifyEcsMappings(indexName);
     }
 
+    private void assertType(String expectedType, Map<String, Object> actualMappings) throws IOException {
+        assertNotNull("expected to get non-null mappings for field", actualMappings);
+        assertEquals(expectedType, actualMappings.get("type"));
+    }
+
     public void testUsage() throws IOException {
         String indexName = "test-usage";
         createTestIndex(indexName);
@@ -205,13 +210,13 @@ public class EcsDynamicTemplatesIT extends ESRestTestCase {
         indexDocument(indexName, fieldsMap);
 
         final Map<String, Object> rawMappings = getMappings(indexName);
-        final Map<String, String> flatFieldMappings = new HashMap<>();
+        final Map<String, Map<String, Object>> flatFieldMappings = new HashMap<>();
         processRawMappingsSubtree(rawMappings, flatFieldMappings, new HashMap<>(), "");
-        assertEquals("scaled_float", flatFieldMappings.get("host.cpu.usage"));
-        assertEquals("scaled_float", flatFieldMappings.get("string.usage"));
-        assertEquals("long", flatFieldMappings.get("usage"));
-        assertEquals("long", flatFieldMappings.get("root.usage.long"));
-        assertEquals("float", flatFieldMappings.get("root.usage.float"));
+        assertType("scaled_float", flatFieldMappings.get("host.cpu.usage"));
+        assertType("scaled_float", flatFieldMappings.get("string.usage"));
+        assertType("long", flatFieldMappings.get("usage"));
+        assertType("long", flatFieldMappings.get("root.usage.long"));
+        assertType("float", flatFieldMappings.get("root.usage.float"));
     }
 
     public void testOnlyMatchLeafFields() throws IOException {
@@ -230,16 +235,16 @@ public class EcsDynamicTemplatesIT extends ESRestTestCase {
         indexDocument(indexName, fieldsMap);
 
         final Map<String, Object> rawMappings = getMappings(indexName);
-        final Map<String, String> flatFieldMappings = new HashMap<>();
+        final Map<String, Map<String, Object>> flatFieldMappings = new HashMap<>();
         processRawMappingsSubtree(rawMappings, flatFieldMappings, new HashMap<>(), "");
-        assertEquals("long", flatFieldMappings.get("foo.message.bar"));
-        assertEquals("long", flatFieldMappings.get("foo.url.path.bar"));
-        assertEquals("long", flatFieldMappings.get("foo.url.full.bar"));
-        assertEquals("long", flatFieldMappings.get("foo.stack_trace.bar"));
-        assertEquals("long", flatFieldMappings.get("foo.user_agent.original.bar"));
-        assertEquals("long", flatFieldMappings.get("foo.created.bar"));
-        assertEquals("float", flatFieldMappings.get("foo._score.bar"));
-        assertEquals("long", flatFieldMappings.get("foo.structured_data"));
+        assertType("long", flatFieldMappings.get("foo.message.bar"));
+        assertType("long", flatFieldMappings.get("foo.url.path.bar"));
+        assertType("long", flatFieldMappings.get("foo.url.full.bar"));
+        assertType("long", flatFieldMappings.get("foo.stack_trace.bar"));
+        assertType("long", flatFieldMappings.get("foo.user_agent.original.bar"));
+        assertType("long", flatFieldMappings.get("foo.created.bar"));
+        assertType("float", flatFieldMappings.get("foo._score.bar"));
+        assertType("long", flatFieldMappings.get("foo.structured_data"));
     }
 
     private static void indexDocument(String indexName, Map<String, Object> flattenedFieldsMap) throws IOException {
@@ -364,28 +369,26 @@ public class EcsDynamicTemplatesIT extends ESRestTestCase {
 
     private void processRawMappingsSubtree(
         final Map<String, Object> fieldSubtrees,
-        final Map<String, String> flatFieldMappings,
-        final Map<String, String> flatMultiFieldsMappings,
+        final Map<String, Map<String, Object>> flatFieldMappings,
+        final Map<String, Map<String, Object>> flatMultiFieldsMappings,
         final String subtreePrefix
     ) {
         fieldSubtrees.forEach((fieldName, fieldMappings) -> {
             String fieldFullPath = subtreePrefix + fieldName;
             Map<String, Object> fieldMappingsMap = ((Map<String, Object>) fieldMappings);
-            String type = (String) fieldMappingsMap.get("type");
-            if (type != null) {
-                flatFieldMappings.put(fieldFullPath, type);
+            if (fieldMappingsMap.get("type") != null) {
+                flatFieldMappings.put(fieldFullPath, fieldMappingsMap);
             }
             Map<String, Object> subfields = (Map<String, Object>) fieldMappingsMap.get("properties");
             if (subfields != null) {
                 processRawMappingsSubtree(subfields, flatFieldMappings, flatMultiFieldsMappings, fieldFullPath + ".");
             }
 
-            Map<String, Map<String, String>> fields = (Map<String, Map<String, String>>) fieldMappingsMap.get("fields");
+            Map<String, Map<String, Object>> fields = (Map<String, Map<String, Object>>) fieldMappingsMap.get("fields");
             if (fields != null) {
                 fields.forEach((subFieldName, multiFieldMappings) -> {
                     String subFieldFullPath = fieldFullPath + "." + subFieldName;
-                    String subFieldType = Objects.requireNonNull(multiFieldMappings.get("type"));
-                    flatMultiFieldsMappings.put(subFieldFullPath, subFieldType);
+                    flatMultiFieldsMappings.put(subFieldFullPath, multiFieldMappings);
                 });
             }
         });
@@ -393,34 +396,44 @@ public class EcsDynamicTemplatesIT extends ESRestTestCase {
 
     private void verifyEcsMappings(String indexName) throws IOException {
         final Map<String, Object> rawMappings = getMappings(indexName);
-        final Map<String, String> flatFieldMappings = new HashMap<>();
-        final Map<String, String> flatMultiFieldsMappings = new HashMap<>();
+        final Map<String, Map<String, Object>> flatFieldMappings = new HashMap<>();
+        final Map<String, Map<String, Object>> flatMultiFieldsMappings = new HashMap<>();
         processRawMappingsSubtree(rawMappings, flatFieldMappings, flatMultiFieldsMappings, "");
 
         Map<String, Map<String, Object>> shallowFieldMapCopy = new HashMap<>(ecsFlatFieldDefinitions);
         logger.info("Testing mapping of {} ECS fields", shallowFieldMapCopy.size());
         List<String> nonEcsFields = new ArrayList<>();
         Map<String, String> fieldToWrongMappingType = new HashMap<>();
-        flatFieldMappings.forEach((fieldName, actualMappingType) -> {
+        List<String> wronglyIndexedFields = new ArrayList<>();
+        List<String> wronglyDocValuedFields = new ArrayList<>();
+        flatFieldMappings.forEach((fieldName, actualMappings) -> {
             Map<String, Object> expectedMappings = shallowFieldMapCopy.remove(fieldName);
             if (expectedMappings == null) {
                 nonEcsFields.add(fieldName);
             } else {
                 String expectedType = (String) expectedMappings.get("type");
+                String actualMappingType = (String) actualMappings.get("type");
                 if (actualMappingType.equals(expectedType) == false) {
                     fieldToWrongMappingType.put(fieldName, actualMappingType);
                 }
+                if (expectedMappings.get("index") != actualMappings.get("index")) {
+                    wronglyIndexedFields.add(fieldName);
+                }
+                if (expectedMappings.get("doc_values") != actualMappings.get("doc_values")) {
+                    wronglyDocValuedFields.add(fieldName);
+                }
             }
         });
 
         Map<String, String> shallowMultiFieldMapCopy = new HashMap<>(ecsFlatMultiFieldDefinitions);
         logger.info("Testing mapping of {} ECS multi-fields", shallowMultiFieldMapCopy.size());
-        flatMultiFieldsMappings.forEach((fieldName, actualMappingType) -> {
+        flatMultiFieldsMappings.forEach((fieldName, actualMappings) -> {
             String expectedType = shallowMultiFieldMapCopy.remove(fieldName);
             if (expectedType != null) {
                 // not finding an entry in the expected multi-field mappings map is acceptable: our dynamic templates are required to
                 // ensure multi-field mapping for all fields with such ECS definitions. However, the patterns in these templates may lead
                 // to multi-field mapping for ECS fields for which such are not defined
+                String actualMappingType = (String) actualMappings.get("type");
                 if (actualMappingType.equals(expectedType) == false) {
                     fieldToWrongMappingType.put(fieldName, actualMappingType);
                 }
@@ -457,6 +470,8 @@ public class EcsDynamicTemplatesIT extends ESRestTestCase {
             );
         });
         nonEcsFields.forEach(field -> logger.error("The test document contains '{}', which is not an ECS field", field));
+        wronglyIndexedFields.forEach(fieldName -> logger.error("ECS field '{}' should be mapped with \"index: false\"", fieldName));
+        wronglyDocValuedFields.forEach(fieldName -> logger.error("ECS field '{}' should be mapped with \"doc_values: false\"", fieldName));
 
         assertTrue("ECS is not fully covered by the current ECS dynamic templates, see details above", shallowFieldMapCopy.isEmpty());
         assertTrue(
@@ -468,5 +483,14 @@ public class EcsDynamicTemplatesIT extends ESRestTestCase {
             fieldToWrongMappingType.isEmpty()
         );
         assertTrue("The test document contains non-ECS fields, see details above", nonEcsFields.isEmpty());
+        assertTrue(
+            "At least one field was not mapped with \"index: false\" as it should according to its ECS definitions, see details above",
+            wronglyIndexedFields.isEmpty()
+        );
+        assertTrue(
+            "At least one field was not mapped with \"doc_values: false\" as it should according to its ECS definitions, see "
+                + "details above",
+            wronglyDocValuedFields.isEmpty()
+        );
     }
 }

x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java

@@ -47,7 +47,7 @@ public class StackTemplateRegistry extends IndexTemplateRegistry {
 
     // The stack template registry version. This number must be incremented when we make changes
     // to built-in templates.
-    public static final int REGISTRY_VERSION = 8;
+    public static final int REGISTRY_VERSION = 9;
 
     public static final String TEMPLATE_VERSION_VARIABLE = "xpack.stack.template.version";
     public static final Setting<Boolean> STACK_TEMPLATES_ENABLED = Setting.boolSetting(