[TEST] re-enable qa tests for mapped runtime fields (#67536)

These tests were left disabled since the introduction of the runtime section, as dynamic templates could not yet create a runtime field. Now that dynamic templates can do that, we re-enable them, though we lose a bit of coverage as we have no ability to mimic the multi-field that's created when string fields are dynamically mapped.

x-pack/plugin/runtime-fields/qa/build.gradle

@@ -43,8 +43,6 @@ subprojects {
         'suggest',
       ]
       if (project.name.equals('core-with-mapped')) {
-        //disabled until runtime fields can be created from a dynamic template
-        enabled = false
         suites += [
           // These two don't support runtime fields on the request. Should they?
           'field_caps',
@@ -57,15 +55,17 @@ subprojects {
       systemProperty 'tests.rest.blacklist',
         [
           /////// TO FIX ///////
-          'search.highlight/40_keyword_ignore/Plain Highligher should skip highlighting ignored keyword values', // The plain highlighter is incompatible with runtime fields. Worth fixing?
+          'search.aggregation/40_range/Date range', //source only date field should also emit values for numbers, it expects strings only
           'search/115_multiple_field_collapsing/two levels fields collapsing', // Broken. Gotta fix.
           'field_caps/30_filter/Field caps with index filter', // We don't support filtering field caps on runtime fields. What should we do?
-          'search.aggregation/10_histogram/*', // runtime doesn't support sub-fields. Maybe it should?
+          'search.aggregation/220_filters_bucket/cache', // runtime keyword does not support split_queries_on_whitespace
           'search/140_pre_filter_search_shards/pre_filter_shard_size with shards that have no hit',
           /////// TO FIX ///////
 
           /////// NOT SUPPORTED ///////
+          'search.highlight/40_keyword_ignore/Plain Highligher should skip highlighting ignored keyword values', // The plain highlighter is incompatible with the prefix queries that we make for runtime fields, use unified highlighter instead.
           'search.aggregation/280_rare_terms/*', // Requires an index and we won't have it
+          'search.aggregation/10_histogram/*', // Runtime fields don't support sub-fields
           // Runtime fields don't have global ords
           'search.aggregation/20_terms/string profiler via global ordinals',
           'search.aggregation/20_terms/Global ordinals are loaded with the global_ordinals execution hint',

x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java

@@ -33,8 +33,8 @@ public class CoreWithMappedRuntimeFieldsIT extends ESClientYamlSuiteTestCase {
 
     private static class MappingRuntimeFieldTranslater extends CoreTestTranslater {
         @Override
-        protected Map<String, Object> dynamicTemplateFor(String type) {
-            return dynamicTemplateToAddRuntimeFields(type);
+        protected Map<String, Object> dynamicTemplateFor() {
+            return dynamicTemplateToAddRuntimeFields();
         }
 
         @Override

x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java

@@ -49,8 +49,8 @@ public class CoreTestsWithSearchRuntimeFieldsIT extends ESClientYamlSuiteTestCas
      */
     private static class SearchRequestRuntimeFieldTranslater extends CoreTestTranslater {
         @Override
-        protected Map<String, Object> dynamicTemplateFor(String type) {
-            return dynamicTemplateToDisableRuntimeCompatibleFields(type);
+        protected Map<String, Object> dynamicTemplateFor() {
+            return dynamicTemplateToDisableRuntimeCompatibleFields();
         }
 
         @Override

x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java

@@ -73,6 +73,7 @@ public abstract class CoreTestTranslater {
 
     protected abstract Suite suite(ClientYamlTestCandidate candidate);
 
+    // TODO geo_points are missing
     private static final Set<String> RUNTIME_TYPES = Set.of(
         BooleanFieldMapper.CONTENT_TYPE,
         DateFieldMapper.CONTENT_TYPE,
@@ -82,15 +83,14 @@ public abstract class CoreTestTranslater {
         NumberType.LONG.typeName()
     );
 
-    protected abstract Map<String, Object> dynamicTemplateFor(String type);
+    protected abstract Map<String, Object> dynamicTemplateFor();
 
-    protected static Map<String, Object> dynamicTemplateToDisableRuntimeCompatibleFields(String type) {
-        return Map.of("type", type, "index", false, "doc_values", false);
+    protected static Map<String, Object> dynamicTemplateToDisableRuntimeCompatibleFields() {
+        return Map.of("mapping", Map.of("index", false, "doc_values", false));
     }
 
-    // TODO there isn't yet a way to create fields in the runtime section from a dynamic template
-    protected static Map<String, Object> dynamicTemplateToAddRuntimeFields(String type) {
-        return Map.ofEntries(Map.entry("type", "runtime"), Map.entry("runtime_type", type));
+    protected static Map<String, Object> dynamicTemplateToAddRuntimeFields() {
+        return Map.of("runtime", Map.of());
     }
 
     protected static Map<String, Object> runtimeFieldLoadingFromSource(String type) {
@@ -109,24 +109,21 @@ public abstract class CoreTestTranslater {
                 Map<String, String> params = Map.of("name", "hack_dynamic_mappings", "create", "true");
                 List<Map<String, Object>> dynamicTemplates = new ArrayList<>();
                 for (String type : RUNTIME_TYPES) {
-                    if (type.equals("ip")) {
-                        // There isn't a dynamic template to pick up ips. They'll just look like strings.
+                    /*
+                     * It would be great to use dynamic:runtime rather than dynamic templates.
+                     * Unfortunately, string gets dynamically mapped as a multi-field (text + keyword) which we can't mimic as
+                     * runtime fields don't support text, and from a dynamic template a field can either be runtime of concrete.
+                     * We would like to define a keyword sub-field under runtime and leave the main field under properties but that
+                     * is not possible. What we do for now is skip strings: we register a dynamic template for each type besides string.
+                     * Ip fields never get dynamically mapped so they'll just look like strings.
+                     */
+                    if (type.equals(IpFieldMapper.CONTENT_TYPE) || type.equals(KeywordFieldMapper.CONTENT_TYPE)) {
                         continue;
                     }
-                    Map<String, Object> mapping = dynamicTemplateFor(type);
-                    if (type.equals("keyword")) {
-                        /*
-                         * For "string"-type dynamic mappings emulate our default
-                         * behavior with a top level text field and a `.keyword`
-                         * multi-field. In our case we disable the keyword field
-                         * and substitute it with an enabled one on the search
-                         * request.
-                         */
-                        mapping = Map.of("type", "text", "fields", Map.of("keyword", mapping));
-                        dynamicTemplates.add(Map.of(type, Map.of("match_mapping_type", "string", "mapping", mapping)));
-                    } else {
-                        dynamicTemplates.add(Map.of(type, Map.of("match_mapping_type", type, "mapping", mapping)));
-                    }
+                    HashMap<String, Object> map = new HashMap<>();
+                    map.put("match_mapping_type", type);
+                    map.putAll(dynamicTemplateFor());
+                    dynamicTemplates.add(Map.of(type, map));
                 }
                 Map<String, Object> indexTemplate = Map.of("settings", Map.of(), "mappings", Map.of("dynamic_templates", dynamicTemplates));
                 List<Map<String, Object>> bodies = List.of(