Generate index.routing_path from dynamic mapping templates (#90552)

Currently, when the `index.routing_path` index setting isn't specified,
then it is auto generated from fields defined in the mapping.

This change also checks for dynamic templates with time series dimension
keyword fields. The `path_match` attributes of these dynamic templates
will then also be selected as routing path.

Closes #90528

docs/changelog/90552.yaml

@@ -0,0 +1,6 @@
+pr: 90552
+summary: "Generate 'index.routing_path' from dynamic mapping templates"
+area: TSDB
+type: enhancement
+issues:
+  - 90528

modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java

@@ -18,6 +18,7 @@ import java.io.IOException;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
@@ -46,6 +47,17 @@ public class TsdbDataStreamRestIT extends ESRestTestCase {
                     }
                 },
                 "mappings":{
+                    "dynamic_templates": [
+                        {
+                            "labels": {
+                                "path_match": "pod.labels.*",
+                                "mapping": {
+                                    "type": "keyword",
+                                    "time_series_dimension": true
+                                }
+                            }
+                        }
+                    ],
                     "properties": {
                         "@timestamp" : {
                             "type": "date"
@@ -212,6 +224,8 @@ public class TsdbDataStreamRestIT extends ESRestTestCase {
         assertThat(startTimeFirstBackingIndex, notNullValue());
         String endTimeFirstBackingIndex = ObjectPath.evaluate(indices, escapedBackingIndex + ".settings.index.time_series.end_time");
         assertThat(endTimeFirstBackingIndex, notNullValue());
+        List<?> routingPaths = ObjectPath.evaluate(indices, escapedBackingIndex + ".settings.index.routing_path");
+        assertThat(routingPaths, containsInAnyOrder("metricset", "k8s.pod.uid", "pod.labels.*"));
 
         var rolloverRequest = new Request("POST", "/k8s/_rollover");
         assertOK(client().performRequest(rolloverRequest));
@@ -333,7 +347,7 @@ public class TsdbDataStreamRestIT extends ESRestTestCase {
         assertThat(ObjectPath.evaluate(responseBody, "template.settings.index.time_series.end_time"), notNullValue());
         assertThat(
             ObjectPath.evaluate(responseBody, "template.settings.index.routing_path"),
-            containsInAnyOrder("metricset", "k8s.pod.uid")
+            containsInAnyOrder("metricset", "k8s.pod.uid", "pod.labels.*")
         );
         assertThat(ObjectPath.evaluate(responseBody, "overlapping"), empty());
     }

modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java

@@ -22,7 +22,10 @@ import org.elasticsearch.index.IndexSettingProvider;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
+import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.index.mapper.MapperBuilderContext;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.MappingParserContext;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
@@ -112,7 +115,7 @@ public class DataStreamIndexSettingsProvider implements IndexSettingProvider {
                     if (allSettings.hasValue(IndexMetadata.INDEX_ROUTING_PATH.getKey()) == false
                         && combinedTemplateMappings.isEmpty() == false) {
                         List<String> routingPaths = findRoutingPaths(indexName, allSettings, combinedTemplateMappings);
-                        if (routingPaths != null) {
+                        if (routingPaths.isEmpty() == false) {
                             builder.putList(INDEX_ROUTING_PATH.getKey(), routingPaths);
                         }
                     }
@@ -159,16 +162,19 @@ public class DataStreamIndexSettingsProvider implements IndexSettingProvider {
                 mapperService.merge(MapperService.SINGLE_MAPPING_NAME, mapping, MapperService.MergeReason.INDEX_TEMPLATE);
             }
 
-            List<String> routingPaths = null;
+            List<String> routingPaths = new ArrayList<>();
             for (var fieldMapper : mapperService.documentMapper().mappers().fieldMappers()) {
-                if (fieldMapper instanceof KeywordFieldMapper keywordFieldMapper) {
-                    if (keywordFieldMapper.fieldType().isDimension()) {
-                        if (routingPaths == null) {
-                            routingPaths = new ArrayList<>();
-                        }
-                        routingPaths.add(keywordFieldMapper.name());
-                    }
-                }
+                extractPath(routingPaths, fieldMapper);
+            }
+            for (var template : mapperService.getAllDynamicTemplates()) {
+                var templateName = "__dynamic__" + template.name();
+                var mappingSnippet = template.mappingForName(templateName, KeywordFieldMapper.CONTENT_TYPE);
+
+                MappingParserContext parserContext = mapperService.parserContext();
+                var mapper = parserContext.typeParser(KeywordFieldMapper.CONTENT_TYPE)
+                    .parse(template.pathMatch(), mappingSnippet, parserContext)
+                    .build(MapperBuilderContext.ROOT);
+                extractPath(routingPaths, mapper);
             }
             return routingPaths;
         } catch (IOException e) {
@@ -176,4 +182,15 @@ public class DataStreamIndexSettingsProvider implements IndexSettingProvider {
         }
     }
 
+    /**
+     * Helper method that adds the name of the mapper to the provided list if it is a keyword dimension field.
+     */
+    private static void extractPath(List<String> routingPaths, Mapper mapper) {
+        if (mapper instanceof KeywordFieldMapper keywordFieldMapper) {
+            if (keywordFieldMapper.fieldType().isDimension()) {
+                routingPaths.add(mapper.name());
+            }
+        }
+    }
+
 }

modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java

@@ -332,4 +332,56 @@ public class DataStreamIndexSettingsProviderTests extends ESTestCase {
         assertThat(result.size(), equalTo(0));
     }
 
+    public void testGenerateRoutingPathFromDynamicTemplate() throws Exception {
+        Metadata metadata = Metadata.EMPTY_METADATA;
+        String dataStreamName = "logs-app1";
+
+        Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS);
+        TimeValue lookAheadTime = TimeValue.timeValueHours(2); // default
+        Settings settings = Settings.EMPTY;
+        String mapping = """
+            {
+                "_doc": {
+                    "dynamic_templates": [
+                        {
+                            "labels": {
+                                "path_match": "prometheus.labels.*",
+                                "mapping": {
+                                    "type": "keyword",
+                                    "time_series_dimension": true
+                                }
+                            }
+                        }
+                    ],
+                    "properties": {
+                        "host": {
+                            "properties": {
+                                "id": {
+                                    "type": "keyword",
+                                    "time_series_dimension": true
+                                }
+                            }
+                        },
+                        "another_field": {
+                            "type": "keyword"
+                        }
+                    }
+                }
+            }
+            """;
+        Settings result = provider.getAdditionalIndexSettings(
+            DataStream.getDefaultBackingIndexName(dataStreamName, 1),
+            dataStreamName,
+            true,
+            metadata,
+            now,
+            settings,
+            List.of(new CompressedXContent(mapping))
+        );
+        assertThat(result.size(), equalTo(3));
+        assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(lookAheadTime.getMillis())));
+        assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(lookAheadTime.getMillis())));
+        assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), containsInAnyOrder("host.id", "prometheus.labels.*"));
+    }
+
 }

server/src/main/java/org/elasticsearch/index/mapper/MapperService.java

@@ -531,4 +531,12 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
         // TODO this should bust the cache somehow. Tracked in https://github.com/elastic/elasticsearch/issues/66722
         return reloadedAnalyzers;
     }
+
+    /**
+     * @return Returns all dynamic templates defined in this mapping.
+     */
+    public DynamicTemplate[] getAllDynamicTemplates() {
+        return documentMapper().mapping().getRoot().dynamicTemplates();
+    }
+
 }