Patterned text conditionally disable templating (#134744)

Follow-up to #134466 to actually check the disable_templating parameter in
the patterned_text mapper. When the parameter is set, values will be stored
as-is in a stored field.

x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextBasicRestIT.java

@@ -7,6 +7,8 @@
 
 package org.elasticsearch.xpack.logsdb.patternedtext;
 
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.common.settings.Settings;
@@ -45,6 +47,17 @@ public class PatternedTextBasicRestIT extends ESRestTestCase {
         return cluster.getHttpAddresses();
     }
 
+    @ParametersFactory(argumentFormatting = "disableTemplating=%b")
+    public static List<Object[]> args() {
+        return List.of(new Object[] { true }, new Object[] { false });
+    }
+
+    private final boolean disableTemplating;
+
+    public PatternedTextBasicRestIT(boolean disableTemplating) {
+        this.disableTemplating = disableTemplating;
+    }
+
     @SuppressWarnings("unchecked")
     public void testBulkInsertThenMatchAllSource() throws IOException {
 
@@ -63,11 +76,12 @@ public class PatternedTextBasicRestIT extends ESRestTestCase {
                             "type": "date"
                         },
                         "message": {
-                            "type": "patterned_text"
+                            "type": "patterned_text",
+                            "disable_templating": %disable_templating%
                         }
                     }
                 }
-            """;
+            """.replace("%disable_templating%", Boolean.toString(disableTemplating));
 
         String indexName = "test-index";
         createIndex(indexName, settings.build(), mapping);

x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapper.java

@@ -30,8 +30,11 @@ import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperBuilderContext;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MappingParserContext;
+import org.elasticsearch.index.mapper.SourceLoader;
+import org.elasticsearch.index.mapper.StringStoredFieldFieldLoader;
 import org.elasticsearch.index.mapper.TextParams;
 import org.elasticsearch.index.mapper.TextSearchInfo;
+import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -261,12 +264,17 @@ public class PatternedTextFieldMapper extends FieldMapper {
             throw new IllegalArgumentException("Multiple values are not allowed for field [" + fieldType().name() + "].");
         }
 
-        // Parse template and args
-        PatternedTextValueProcessor.Parts parts = PatternedTextValueProcessor.split(value);
-
         // Add index on original value
         context.doc().add(new Field(fieldType().name(), value, fieldType));
 
+        if (fieldType().disableTemplating()) {
+            context.doc().add(new StoredField(fieldType().storedNamed(), new BytesRef(value)));
+            return;
+        }
+
+        // Parse template and args
+        PatternedTextValueProcessor.Parts parts = PatternedTextValueProcessor.split(value);
+
         // Add template_id doc_values
         context.doc().add(templateIdMapper.buildKeywordField(new BytesRef(parts.templateId())));
 
@@ -305,14 +313,25 @@ public class PatternedTextFieldMapper extends FieldMapper {
 
     @Override
     protected SyntheticSourceSupport syntheticSourceSupport() {
-        return new SyntheticSourceSupport.Native(
-            () -> new CompositeSyntheticFieldLoader(
-                leafName(),
-                fullPath(),
-                new PatternedTextSyntheticFieldLoaderLayer(
-                    fieldType().name(),
-                    leafReader -> PatternedTextCompositeValues.from(leafReader, fieldType())
-                )
+        return new SyntheticSourceSupport.Native(this::getSyntheticFieldLoader);
+    }
+
+    private SourceLoader.SyntheticFieldLoader getSyntheticFieldLoader() {
+        if (fieldType().disableTemplating()) {
+            return new StringStoredFieldFieldLoader(fieldType().storedNamed(), fieldType().name(), leafName()) {
+                @Override
+                protected void write(XContentBuilder b, Object value) throws IOException {
+                    b.value(((BytesRef) value).utf8ToString());
+                }
+            };
+        }
+
+        return new CompositeSyntheticFieldLoader(
+            leafName(),
+            fullPath(),
+            new PatternedTextSyntheticFieldLoaderLayer(
+                fieldType().name(),
+                leafReader -> PatternedTextCompositeValues.from(leafReader, fieldType())
             )
         );
     }

x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldType.java

@@ -29,7 +29,9 @@ import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.index.fielddata.FieldDataContext;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.SourceValueFetcherSortedBinaryIndexFieldData;
+import org.elasticsearch.index.fieldvisitor.StoredFieldLoader;
 import org.elasticsearch.index.mapper.BlockLoader;
+import org.elasticsearch.index.mapper.BlockStoredFieldsReader;
 import org.elasticsearch.index.mapper.SourceValueFetcher;
 import org.elasticsearch.index.mapper.StringFieldType;
 import org.elasticsearch.index.mapper.TextFieldMapper;
@@ -49,6 +51,7 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 
 public class PatternedTextFieldType extends StringFieldType {
 
@@ -118,6 +121,10 @@ public class PatternedTextFieldType extends StringFieldType {
     private IOFunction<LeafReaderContext, CheckedIntFunction<List<Object>, IOException>> getValueFetcherProvider(
         SearchExecutionContext searchExecutionContext
     ) {
+        if (disableTemplating) {
+            return storedFieldFetcher(storedNamed());
+        }
+
         return context -> {
             ValueFetcher valueFetcher = valueFetcher(searchExecutionContext, null);
             SourceProvider sourceProvider = searchExecutionContext.lookup();
@@ -132,6 +139,18 @@ public class PatternedTextFieldType extends StringFieldType {
         };
     }
 
+    private static IOFunction<LeafReaderContext, CheckedIntFunction<List<Object>, IOException>> storedFieldFetcher(String name) {
+        var loader = StoredFieldLoader.create(false, Set.of(name));
+        return context -> {
+            var leafLoader = loader.getLoader(context, null);
+            return docId -> {
+                leafLoader.advanceTo(docId);
+                var storedFields = leafLoader.storedFields();
+                return storedFields.get(name);
+            };
+        };
+    }
+
     private Query maybeSourceConfirmQuery(Query query, SearchExecutionContext context) {
         // Disable scoring similarly to match_only_text
         if (hasPositions) {
@@ -262,6 +281,10 @@ public class PatternedTextFieldType extends StringFieldType {
 
     @Override
     public BlockLoader blockLoader(BlockLoaderContext blContext) {
+        if (disableTemplating) {
+            return new BlockStoredFieldsReader.BytesFromBytesRefsBlockLoader(storedNamed());
+        }
+
         return new PatternedTextBlockLoader((leafReader -> PatternedTextCompositeValues.from(leafReader, this)));
     }
 

x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapperTests.java

@@ -97,6 +97,16 @@ public class PatternedTextFieldMapperTests extends MapperTestCase {
         assertPhraseQuery(createSytheticSourceMapperService(fieldMapping(b -> b.field("type", "patterned_text"))));
     }
 
+    public void testPhraseQueryStandardSourceDisableTemplating() throws IOException {
+        assertPhraseQuery(createMapperService(fieldMapping(b -> b.field("type", "patterned_text").field("disable_templating", true))));
+    }
+
+    public void testPhraseQuerySyntheticSourceDisableTemplating() throws IOException {
+        assertPhraseQuery(
+            createSytheticSourceMapperService(fieldMapping(b -> b.field("type", "patterned_text").field("disable_templating", true)))
+        );
+    }
+
     private void assertPhraseQuery(MapperService mapperService) throws IOException {
         try (Directory directory = newDirectory()) {
             RandomIndexWriter iw = new RandomIndexWriter(random(), directory);
@@ -322,6 +332,9 @@ public class PatternedTextFieldMapperTests extends MapperTestCase {
 
         private void mapping(XContentBuilder b) throws IOException {
             b.field("type", "patterned_text");
+            if (randomBoolean()) {
+                b.field("disable_templating", true);
+            }
         }
 
         @Override

x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextIntegrationTests.java

@@ -7,6 +7,8 @@
 
 package org.elasticsearch.xpack.logsdb.patternedtext;
 
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.DocWriteRequest;
@@ -41,6 +43,7 @@ import java.time.ZonedDateTime;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -56,6 +59,27 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFa
 public class PatternedTextIntegrationTests extends ESSingleNodeTestCase {
     private static final Logger logger = LogManager.getLogger(PatternedTextIntegrationTests.class);
 
+    @ParametersFactory(argumentFormatting = "indexOptions=%s, disableTemplating=%b")
+    public static List<Object[]> args() {
+        List<Object[]> args = new ArrayList<>();
+        for (var indexOption : new String[] { "docs", "positions" }) {
+            for (var templating : new boolean[] { true, false }) {
+                args.add(new Object[] { indexOption, templating });
+            }
+        }
+        return Collections.unmodifiableList(args);
+    }
+
+    private final String indexOptions;
+    private final boolean disableTemplating;
+    private final String mapping;
+
+    public PatternedTextIntegrationTests(String indexOptions, boolean disableTemplating) {
+        this.indexOptions = indexOptions;
+        this.disableTemplating = disableTemplating;
+        this.mapping = getMapping(indexOptions, disableTemplating);
+    }
+
     @Override
     protected Settings nodeSettings() {
         return Settings.builder().put(LicenseSettings.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial").build();
@@ -75,17 +99,15 @@ public class PatternedTextIntegrationTests extends ESSingleNodeTestCase {
                 "@timestamp": { "type": "date" },
                 "field_match_only_text": { "type": "match_only_text" },
                 "field_patterned_text": {
-                    "type": "patterned_text",
-                    "index_options": "%",
-                    "analyzer": "standard"
+                  "type": "patterned_text",
+                  "index_options": "%index_options%",
+                  "disable_templating": "%disable_templating%",
+                  "analyzer": "standard"
                 }
               }
             }
         """;
 
-    private static final String MAPPING_DOCS_ONLY = MAPPING_TEMPLATE.replace("%", "docs");
-    private static final String MAPPING_POSITIONS = MAPPING_TEMPLATE.replace("%", "positions");
-
     private static final Settings LOGSDB_SETTING = Settings.builder().put(IndexSettings.MODE.getKey(), "logsdb").build();
 
     @Before
@@ -100,8 +122,12 @@ public class PatternedTextIntegrationTests extends ESSingleNodeTestCase {
         }
     }
 
+    private String getMapping(String indexOptions, boolean disableTemplating) {
+        return MAPPING_TEMPLATE.replace("%index_options%", indexOptions)
+            .replace("%disable_templating%", Boolean.toString(disableTemplating));
+    }
+
     public void testSourceMatchAllManyValues() throws IOException {
-        var mapping = randomBoolean() ? MAPPING_DOCS_ONLY : MAPPING_POSITIONS;
         var createRequest = indicesAdmin().prepareCreate(INDEX).setSettings(LOGSDB_SETTING).setMapping(mapping);
         createIndex(INDEX, createRequest);
 
@@ -114,7 +140,6 @@ public class PatternedTextIntegrationTests extends ESSingleNodeTestCase {
     }
 
     public void testLargeValueIsStored() throws IOException {
-        var mapping = randomBoolean() ? MAPPING_DOCS_ONLY : MAPPING_POSITIONS;
         var createRequest = indicesAdmin().prepareCreate(INDEX).setSettings(LOGSDB_SETTING).setMapping(mapping);
         IndexService indexService = createIndex(INDEX, createRequest);
 
@@ -136,7 +161,6 @@ public class PatternedTextIntegrationTests extends ESSingleNodeTestCase {
     }
 
     public void testSmallValueNotStored() throws IOException {
-        var mapping = randomBoolean() ? MAPPING_DOCS_ONLY : MAPPING_POSITIONS;
         var createRequest = indicesAdmin().prepareCreate(INDEX).setSettings(LOGSDB_SETTING).setMapping(mapping);
         IndexService indexService = createIndex(INDEX, createRequest);
 
@@ -148,17 +172,20 @@ public class PatternedTextIntegrationTests extends ESSingleNodeTestCase {
         assertMappings();
         assertMessagesInSource(messages);
 
-        // assert does not contain stored field
+        // assert only contains stored field if templating is disabled
         try (var searcher = indexService.getShard(0).acquireSearcher(INDEX)) {
             try (var indexReader = searcher.getIndexReader()) {
                 var document = indexReader.storedFields().document(0);
-                assertNull(document.getField("field_patterned_text.stored"));
+                if (disableTemplating) {
+                    assertEquals(document.getField("field_patterned_text.stored").binaryValue().utf8ToString(), message);
+                } else {
+                    assertNull(document.getField("field_patterned_text.stored"));
+                }
             }
         }
     }
 
     public void testQueryResultsSameAsMatchOnlyText() throws IOException {
-        var mapping = randomBoolean() ? MAPPING_DOCS_ONLY : MAPPING_POSITIONS;
         var createRequest = new CreateIndexRequest(INDEX).mapping(mapping);
 
         if (randomBoolean()) {