[8.x] Guard second doc parsing pass with index setting (#114649) (#114799)

* Guard second doc parsing pass with index setting (#114649)

* Guard second doc parsing pass with index setting

* add test

* updates

* updates

* merge

(cherry picked from commit 98e0a4e953d339402c87ed426a70e6cc8320c17f)

* Update 21_synthetic_source_stored.yml

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml

@@ -411,6 +411,55 @@ index param - nested array within array:
   - match: { hits.hits.0._source.path.to.some.3.id: [ 1000, 2000 ] }
 
 
+---
+index param - nested array within array - disabled second pass:
+  - requires:
+      cluster_features: ["mapper.synthetic_source_keep"]
+      reason: requires tracking ignored source
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            index:
+              synthetic_source:
+                enable_second_doc_parsing_pass: false
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              name:
+                type: keyword
+              path:
+                properties:
+                  to:
+                    properties:
+                      some:
+                        synthetic_source_keep: arrays
+                        properties:
+                          id:
+                            type: integer
+
+  - do:
+      bulk:
+        index: test
+        refresh: true
+        body:
+          - '{ "create": { } }'
+          - '{ "name": "A", "path": [ { "to": [ { "some" : [ { "id": 10 }, { "id": [1, 3, 2] } ] }, { "some": { "id": 100 } } ] }, { "to": { "some": { "id": [1000, 2000] } } } ] }'
+  - match: { errors: false }
+
+  - do:
+      search:
+        index: test
+        sort: name
+  - match:  { hits.hits.0._source.name: A }
+  - length: { hits.hits.0._source.path.to.some: 2}
+  - match: { hits.hits.0._source.path.to.some.0.id: 10 }
+  - match: { hits.hits.0._source.path.to.some.1.id: [ 1, 3, 2] }
+
+
 ---
 # 112156
 stored field under object with store_array_source:

server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java

@@ -188,6 +188,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
         FieldMapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING,
         IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING,
         IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING,
+        IndexSettings.SYNTHETIC_SOURCE_SECOND_DOC_PARSING_PASS_SETTING,
         SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING,
 
         // validate that built-in similarities don't get redefined

server/src/main/java/org/elasticsearch/index/IndexSettings.java

@@ -655,6 +655,13 @@ public final class IndexSettings {
         Property.Final
     );
 
+    public static final Setting<Boolean> SYNTHETIC_SOURCE_SECOND_DOC_PARSING_PASS_SETTING = Setting.boolSetting(
+        "index.synthetic_source.enable_second_doc_parsing_pass",
+        true,
+        Property.IndexScope,
+        Property.Dynamic
+    );
+
     /**
      * Returns <code>true</code> if TSDB encoding is enabled. The default is <code>true</code>
      */
@@ -808,6 +815,7 @@ public final class IndexSettings {
     private volatile long mappingDimensionFieldsLimit;
     private volatile boolean skipIgnoredSourceWrite;
     private volatile boolean skipIgnoredSourceRead;
+    private volatile boolean syntheticSourceSecondDocParsingPassEnabled;
     private final SourceFieldMapper.Mode indexMappingSourceMode;
 
     /**
@@ -969,6 +977,7 @@ public final class IndexSettings {
         es87TSDBCodecEnabled = scopedSettings.get(TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING);
         skipIgnoredSourceWrite = scopedSettings.get(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING);
         skipIgnoredSourceRead = scopedSettings.get(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING);
+        syntheticSourceSecondDocParsingPassEnabled = scopedSettings.get(SYNTHETIC_SOURCE_SECOND_DOC_PARSING_PASS_SETTING);
         indexMappingSourceMode = scopedSettings.get(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING);
 
         scopedSettings.addSettingsUpdateConsumer(
@@ -1057,6 +1066,10 @@ public final class IndexSettings {
             this::setSkipIgnoredSourceWrite
         );
         scopedSettings.addSettingsUpdateConsumer(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING, this::setSkipIgnoredSourceRead);
+        scopedSettings.addSettingsUpdateConsumer(
+            SYNTHETIC_SOURCE_SECOND_DOC_PARSING_PASS_SETTING,
+            this::setSyntheticSourceSecondDocParsingPassEnabled
+        );
     }
 
     private void setSearchIdleAfter(TimeValue searchIdleAfter) {
@@ -1649,6 +1662,14 @@ public final class IndexSettings {
         this.skipIgnoredSourceRead = value;
     }
 
+    private void setSyntheticSourceSecondDocParsingPassEnabled(boolean syntheticSourceSecondDocParsingPassEnabled) {
+        this.syntheticSourceSecondDocParsingPassEnabled = syntheticSourceSecondDocParsingPassEnabled;
+    }
+
+    public boolean isSyntheticSourceSecondDocParsingPassEnabled() {
+        return syntheticSourceSecondDocParsingPassEnabled;
+    }
+
     public SourceFieldMapper.Mode getIndexMappingSourceMode() {
         return indexMappingSourceMode;
     }

server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java

@@ -111,6 +111,7 @@ public abstract class DocumentParserContext {
     private final Set<String> ignoredFields;
     private final List<IgnoredSourceFieldMapper.NameValue> ignoredFieldValues;
     private final List<IgnoredSourceFieldMapper.NameValue> ignoredFieldsMissingValues;
+    private final boolean inArrayScopeEnabled;
     private boolean inArrayScope;
 
     private final Map<String, List<Mapper>> dynamicMappers;
@@ -143,6 +144,7 @@ public abstract class DocumentParserContext {
         Set<String> ignoreFields,
         List<IgnoredSourceFieldMapper.NameValue> ignoredFieldValues,
         List<IgnoredSourceFieldMapper.NameValue> ignoredFieldsWithNoSource,
+        boolean inArrayScopeEnabled,
         boolean inArrayScope,
         Map<String, List<Mapper>> dynamicMappers,
         Map<String, ObjectMapper> dynamicObjectMappers,
@@ -164,6 +166,7 @@ public abstract class DocumentParserContext {
         this.ignoredFields = ignoreFields;
         this.ignoredFieldValues = ignoredFieldValues;
         this.ignoredFieldsMissingValues = ignoredFieldsWithNoSource;
+        this.inArrayScopeEnabled = inArrayScopeEnabled;
         this.inArrayScope = inArrayScope;
         this.dynamicMappers = dynamicMappers;
         this.dynamicObjectMappers = dynamicObjectMappers;
@@ -188,6 +191,7 @@ public abstract class DocumentParserContext {
             in.ignoredFields,
             in.ignoredFieldValues,
             in.ignoredFieldsMissingValues,
+            in.inArrayScopeEnabled,
             in.inArrayScope,
             in.dynamicMappers,
             in.dynamicObjectMappers,
@@ -219,6 +223,7 @@ public abstract class DocumentParserContext {
             new HashSet<>(),
             new ArrayList<>(),
             new ArrayList<>(),
+            mappingParserContext.getIndexSettings().isSyntheticSourceSecondDocParsingPassEnabled(),
             false,
             new HashMap<>(),
             new HashMap<>(),
@@ -371,7 +376,7 @@ public abstract class DocumentParserContext {
      * Applies to synthetic source only.
      */
     public final DocumentParserContext maybeCloneForArray(Mapper mapper) throws IOException {
-        if (canAddIgnoredField() && mapper instanceof ObjectMapper) {
+        if (canAddIgnoredField() && mapper instanceof ObjectMapper && inArrayScopeEnabled) {
             boolean isNested = mapper instanceof NestedObjectMapper;
             if ((inArrayScope == false && isNested == false) || (inArrayScope && isNested)) {
                 DocumentParserContext subcontext = switchParser(parser());

x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java

@@ -333,6 +333,7 @@ public class TransportResumeFollowActionTests extends ESTestCase {
         replicatedSettings.add(IndexSettings.MAX_SHINGLE_DIFF_SETTING);
         replicatedSettings.add(IndexSettings.TIME_SERIES_END_TIME);
         replicatedSettings.add(IndexSettings.PREFER_ILM_SETTING);
+        replicatedSettings.add(IndexSettings.SYNTHETIC_SOURCE_SECOND_DOC_PARSING_PASS_SETTING);
         replicatedSettings.add(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING);
         replicatedSettings.add(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING);
         replicatedSettings.add(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING);