Make scripted search templates work with new mediaType from XContentType.JSON (#67677)

Stored scripts can have content_type option set, however when empty they default to XContentType.JSON#mediaType(). Commit 5e74f79 has changed this in master (ES8) method to return application/json;charset=utf-8 (previously application/json; charset=UTF-8)
This means that when upgrading ES from version 7 to 8 stored script will fail when being used as the encoder is being matched with string equality (map key)

This commit address this by adding back (in addition) the old application/json; charset=UTF-8 into the encoders map.

closes #66986

modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomMustacheFactory.java

@@ -30,7 +30,6 @@ import com.github.mustachejava.TemplateContext;
 import com.github.mustachejava.codes.DefaultMustache;
 import com.github.mustachejava.codes.IterableCode;
 import com.github.mustachejava.codes.WriteCode;
-import org.apache.lucene.search.highlight.DefaultEncoder;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -51,30 +50,31 @@ import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 public class CustomMustacheFactory extends DefaultMustacheFactory {
+    static final String V7_JSON_MEDIA_TYPE_WITH_CHARSET = "application/json; charset=UTF-8";
+    static final String JSON_MEDIA_TYPE_WITH_CHARSET = "application/json;charset=utf-8";
+    static final String JSON_MEDIA_TYPE = "application/json";
+    static final String PLAIN_TEXT_MEDIA_TYPE = "text/plain";
+    static final String X_WWW_FORM_URLENCODED_MEDIA_TYPE = "application/x-www-form-urlencoded";
 
-    static final String JSON_MIME_TYPE_WITH_CHARSET = "application/json;charset=utf-8";
-    static final String JSON_MIME_TYPE = "application/json";
-    static final String PLAIN_TEXT_MIME_TYPE = "text/plain";
-    static final String X_WWW_FORM_URLENCODED_MIME_TYPE = "application/x-www-form-urlencoded";
-
-    private static final String DEFAULT_MIME_TYPE = JSON_MIME_TYPE;
+    private static final String DEFAULT_MEDIA_TYPE = JSON_MEDIA_TYPE;
 
     private static final Map<String, Supplier<Encoder>> ENCODERS = Map.of(
-            JSON_MIME_TYPE_WITH_CHARSET, JsonEscapeEncoder::new,
-            JSON_MIME_TYPE, JsonEscapeEncoder::new,
-            PLAIN_TEXT_MIME_TYPE, DefaultEncoder::new,
-            X_WWW_FORM_URLENCODED_MIME_TYPE, UrlEncoder::new);
+        V7_JSON_MEDIA_TYPE_WITH_CHARSET, JsonEscapeEncoder::new,
+        JSON_MEDIA_TYPE_WITH_CHARSET, JsonEscapeEncoder::new,
+        JSON_MEDIA_TYPE, JsonEscapeEncoder::new,
+        PLAIN_TEXT_MEDIA_TYPE, DefaultEncoder::new,
+        X_WWW_FORM_URLENCODED_MEDIA_TYPE, UrlEncoder::new);
 
     private final Encoder encoder;
 
-    public CustomMustacheFactory(String mimeType) {
+    public CustomMustacheFactory(String mediaType) {
         super();
         setObjectHandler(new CustomReflectionObjectHandler());
-        this.encoder = createEncoder(mimeType);
+        this.encoder = createEncoder(mediaType);
     }
 
     public CustomMustacheFactory() {
-        this(DEFAULT_MIME_TYPE);
+        this(DEFAULT_MEDIA_TYPE);
     }
 
     @Override
@@ -86,10 +86,10 @@ public class CustomMustacheFactory extends DefaultMustacheFactory {
         }
     }
 
-    static Encoder createEncoder(String mimeType) {
-        final Supplier<Encoder> supplier = ENCODERS.get(mimeType);
+    static Encoder createEncoder(String mediaType) {
+        final Supplier<Encoder> supplier = ENCODERS.get(mediaType);
         if (supplier == null) {
-            throw new IllegalArgumentException("No encoder found for MIME type [" + mimeType + "]");
+            throw new IllegalArgumentException("No encoder found for media type [" + mediaType + "]");
         }
         return supplier.get();
     }

modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/CustomMustacheFactoryTests.java

@@ -28,9 +28,9 @@ import java.util.Map;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.singletonMap;
-import static org.elasticsearch.script.mustache.CustomMustacheFactory.JSON_MIME_TYPE;
-import static org.elasticsearch.script.mustache.CustomMustacheFactory.PLAIN_TEXT_MIME_TYPE;
-import static org.elasticsearch.script.mustache.CustomMustacheFactory.X_WWW_FORM_URLENCODED_MIME_TYPE;
+import static org.elasticsearch.script.mustache.CustomMustacheFactory.JSON_MEDIA_TYPE;
+import static org.elasticsearch.script.mustache.CustomMustacheFactory.PLAIN_TEXT_MEDIA_TYPE;
+import static org.elasticsearch.script.mustache.CustomMustacheFactory.X_WWW_FORM_URLENCODED_MEDIA_TYPE;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 
@@ -40,33 +40,33 @@ public class CustomMustacheFactoryTests extends ESTestCase {
         {
             final IllegalArgumentException e =
                     expectThrows(IllegalArgumentException.class, () -> CustomMustacheFactory.createEncoder("non-existent"));
-            assertThat(e.getMessage(), equalTo("No encoder found for MIME type [non-existent]"));
+            assertThat(e.getMessage(), equalTo("No encoder found for media type [non-existent]"));
         }
 
         {
             final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> CustomMustacheFactory.createEncoder(""));
-            assertThat(e.getMessage(), equalTo("No encoder found for MIME type []"));
+            assertThat(e.getMessage(), equalTo("No encoder found for media type []"));
         }
 
         {
             final IllegalArgumentException e =
                     expectThrows(IllegalArgumentException.class, () -> CustomMustacheFactory.createEncoder("test"));
-            assertThat(e.getMessage(), equalTo("No encoder found for MIME type [test]"));
+            assertThat(e.getMessage(), equalTo("No encoder found for media type [test]"));
         }
 
-        assertThat(CustomMustacheFactory.createEncoder(CustomMustacheFactory.JSON_MIME_TYPE_WITH_CHARSET),
+        assertThat(CustomMustacheFactory.createEncoder(CustomMustacheFactory.JSON_MEDIA_TYPE_WITH_CHARSET),
             instanceOf(CustomMustacheFactory.JsonEscapeEncoder.class));
-        assertThat(CustomMustacheFactory.createEncoder(CustomMustacheFactory.JSON_MIME_TYPE),
+        assertThat(CustomMustacheFactory.createEncoder(CustomMustacheFactory.JSON_MEDIA_TYPE),
                 instanceOf(CustomMustacheFactory.JsonEscapeEncoder.class));
-        assertThat(CustomMustacheFactory.createEncoder(CustomMustacheFactory.PLAIN_TEXT_MIME_TYPE),
+        assertThat(CustomMustacheFactory.createEncoder(CustomMustacheFactory.PLAIN_TEXT_MEDIA_TYPE),
                 instanceOf(CustomMustacheFactory.DefaultEncoder.class));
-        assertThat(CustomMustacheFactory.createEncoder(CustomMustacheFactory.X_WWW_FORM_URLENCODED_MIME_TYPE),
+        assertThat(CustomMustacheFactory.createEncoder(CustomMustacheFactory.X_WWW_FORM_URLENCODED_MEDIA_TYPE),
                 instanceOf(CustomMustacheFactory.UrlEncoder.class));
     }
 
     public void testJsonEscapeEncoder() {
         final ScriptEngine engine = new MustacheScriptEngine();
-        final Map<String, String> params = randomBoolean() ? singletonMap(Script.CONTENT_TYPE_OPTION, JSON_MIME_TYPE) : emptyMap();
+        final Map<String, String> params = randomBoolean() ? singletonMap(Script.CONTENT_TYPE_OPTION, JSON_MEDIA_TYPE) : emptyMap();
 
         TemplateScript.Factory compiled = engine.compile(null, "{\"field\": \"{{value}}\"}", TemplateScript.CONTEXT, params);
 
@@ -76,7 +76,7 @@ public class CustomMustacheFactoryTests extends ESTestCase {
 
     public void testDefaultEncoder() {
         final ScriptEngine engine = new MustacheScriptEngine();
-        final Map<String, String> params = singletonMap(Script.CONTENT_TYPE_OPTION, PLAIN_TEXT_MIME_TYPE);
+        final Map<String, String> params = singletonMap(Script.CONTENT_TYPE_OPTION, PLAIN_TEXT_MEDIA_TYPE);
 
         TemplateScript.Factory compiled = engine.compile(null, "{\"field\": \"{{value}}\"}", TemplateScript.CONTEXT, params);
 
@@ -86,7 +86,7 @@ public class CustomMustacheFactoryTests extends ESTestCase {
 
     public void testUrlEncoder() {
         final ScriptEngine engine = new MustacheScriptEngine();
-        final Map<String, String> params = singletonMap(Script.CONTENT_TYPE_OPTION, X_WWW_FORM_URLENCODED_MIME_TYPE);
+        final Map<String, String> params = singletonMap(Script.CONTENT_TYPE_OPTION, X_WWW_FORM_URLENCODED_MEDIA_TYPE);
 
         TemplateScript.Factory compiled = engine.compile(null, "{\"field\": \"{{value}}\"}", TemplateScript.CONTEXT, params);
 

qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml

@@ -1,8 +1,5 @@
 ---
 "Verify that we can still find things with the template":
-  - skip:
-      version: "all"
-      reason: "Awaits fix: https://github.com/elastic/elasticsearch/issues/66986"
 
   - do:
       search_template:

qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml

@@ -1,8 +1,5 @@
 ---
 "Verify that we can still find things with the template":
-  - skip:
-      version: "all"
-      reason: "Awaits fix: https://github.com/elastic/elasticsearch/issues/66986"
 
   - do:
       search_template: