Add option to skip using _ignored_source field for synthetic source (#112963) (#113057)

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java

@@ -33,6 +33,7 @@ import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 import org.elasticsearch.index.engine.EngineConfig;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.index.store.FsDirectoryFactory;
@@ -183,6 +184,8 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
         IndexSettings.PREFER_ILM_SETTING,
         DataStreamFailureStoreDefinition.FAILURE_STORE_DEFINITION_VERSION_SETTING,
         FieldMapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING,
+        IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING,
+        IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING,
 
         // validate that built-in similarities don't get redefined
         Setting.groupSetting("index.similarity.", (s) -> {

server/src/main/java/org/elasticsearch/index/IndexSettings.java

@@ -25,6 +25,7 @@ import org.elasticsearch.common.time.DateUtils;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.ingest.IngestService;
@@ -778,6 +779,8 @@ public final class IndexSettings {
     private volatile long mappingDepthLimit;
     private volatile long mappingFieldNameLengthLimit;
     private volatile long mappingDimensionFieldsLimit;
+    private volatile boolean skipIgnoredSourceWrite;
+    private volatile boolean skipIgnoredSourceRead;
 
     /**
      * The maximum number of refresh listeners allows on this shard.
@@ -936,6 +939,8 @@ public final class IndexSettings {
         indexRouting = IndexRouting.fromIndexMetadata(indexMetadata);
         sourceKeepMode = scopedSettings.get(Mapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING);
         es87TSDBCodecEnabled = scopedSettings.get(TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING);
+        skipIgnoredSourceWrite = scopedSettings.get(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING);
+        skipIgnoredSourceRead = scopedSettings.get(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING);
 
         scopedSettings.addSettingsUpdateConsumer(
             MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING,
@@ -1018,6 +1023,11 @@ public final class IndexSettings {
         scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_DEPTH_LIMIT_SETTING, this::setMappingDepthLimit);
         scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING, this::setMappingFieldNameLengthLimit);
         scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING, this::setMappingDimensionFieldsLimit);
+        scopedSettings.addSettingsUpdateConsumer(
+            IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING,
+            this::setSkipIgnoredSourceWrite
+        );
+        scopedSettings.addSettingsUpdateConsumer(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING, this::setSkipIgnoredSourceRead);
     }
 
     private void setSearchIdleAfter(TimeValue searchIdleAfter) {
@@ -1594,6 +1604,22 @@ public final class IndexSettings {
         this.mappingDimensionFieldsLimit = value;
     }
 
+    public boolean getSkipIgnoredSourceWrite() {
+        return skipIgnoredSourceWrite;
+    }
+
+    private void setSkipIgnoredSourceWrite(boolean value) {
+        this.skipIgnoredSourceWrite = value;
+    }
+
+    public boolean getSkipIgnoredSourceRead() {
+        return skipIgnoredSourceRead;
+    }
+
+    private void setSkipIgnoredSourceRead(boolean value) {
+        this.skipIgnoredSourceRead = value;
+    }
+
     /**
      * The bounds for {@code @timestamp} on this index or
      * {@code null} if there are no bounds.

server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java

@@ -333,7 +333,7 @@ public abstract class DocumentParserContext {
     }
 
     public final boolean canAddIgnoredField() {
-        return mappingLookup.isSourceSynthetic() && clonedSource == false;
+        return mappingLookup.isSourceSynthetic() && clonedSource == false && indexSettings().getSkipIgnoredSourceWrite() == false;
     }
 
     Mapper.SourceKeepMode sourceKeepModeFromIndexSettings() {
@@ -367,7 +367,7 @@ public abstract class DocumentParserContext {
 
     public void markFieldAsCopyTo(String fieldName) {
         copyToFields.add(fieldName);
-        if (mappingLookup.isSourceSynthetic()) {
+        if (mappingLookup.isSourceSynthetic() && indexSettings().getSkipIgnoredSourceWrite() == false) {
             /*
             Mark this field as containing copied data meaning it should not be present
             in synthetic _source (to be consistent with stored _source).

server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java

@@ -10,12 +10,15 @@
 package org.elasticsearch.index.mapper;
 
 import org.apache.lucene.document.StoredField;
+import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.util.ByteUtils;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.features.NodeFeature;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentType;
@@ -27,6 +30,8 @@ import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.stream.Stream;
 
 /**
 
@@ -40,6 +45,7 @@ import java.util.Map;
  * if we can replace it for all use cases to avoid duplication, assuming that the storage tradeoff is favorable.
  */
 public class IgnoredSourceFieldMapper extends MetadataFieldMapper {
+    private final IndexSettings indexSettings;
 
     // This factor is used to combine two offsets within the same integer:
     // - the offset of the end of the parent field within the field name (N / PARENT_OFFSET_IN_NAME_OFFSET)
@@ -49,12 +55,32 @@ public class IgnoredSourceFieldMapper extends MetadataFieldMapper {
 
     public static final String NAME = "_ignored_source";
 
-    public static final IgnoredSourceFieldMapper INSTANCE = new IgnoredSourceFieldMapper();
-
-    public static final TypeParser PARSER = new FixedTypeParser(context -> INSTANCE);
+    public static final TypeParser PARSER = new FixedTypeParser(context -> new IgnoredSourceFieldMapper(context.getIndexSettings()));
 
     static final NodeFeature TRACK_IGNORED_SOURCE = new NodeFeature("mapper.track_ignored_source");
 
+    /*
+        Setting to disable encoding and writing values for this field.
+        This is needed to unblock index functionality in case there is a bug on this code path.
+     */
+    public static final Setting<Boolean> SKIP_IGNORED_SOURCE_WRITE_SETTING = Setting.boolSetting(
+        "index.mapping.synthetic_source.skip_ignored_source_write",
+        false,
+        Setting.Property.Dynamic,
+        Setting.Property.IndexScope
+    );
+
+    /*
+        Setting to disable reading and decoding values stored in this field.
+        This is needed to unblock search functionality in case there is a bug on this code path.
+     */
+    public static final Setting<Boolean> SKIP_IGNORED_SOURCE_READ_SETTING = Setting.boolSetting(
+        "index.mapping.synthetic_source.skip_ignored_source_read",
+        false,
+        Setting.Property.Dynamic,
+        Setting.Property.IndexScope
+    );
+
     /*
      * Container for the ignored field data:
      *  - the full name
@@ -108,8 +134,9 @@ public class IgnoredSourceFieldMapper extends MetadataFieldMapper {
         }
     }
 
-    private IgnoredSourceFieldMapper() {
+    private IgnoredSourceFieldMapper(IndexSettings indexSettings) {
         super(IgnoredValuesFieldMapperType.INSTANCE);
+        this.indexSettings = indexSettings;
     }
 
     @Override
@@ -151,6 +178,64 @@ public class IgnoredSourceFieldMapper extends MetadataFieldMapper {
         return new NameValue(name, parentOffset, value, null);
     }
 
+    // In rare cases decoding values stored in this field can fail leading to entire source
+    // not being available.
+    // We would like to have an option to lose some values in synthetic source
+    // but have search not fail.
+    public static Set<String> ensureLoaded(Set<String> fieldsToLoadForSyntheticSource, IndexSettings indexSettings) {
+        if (indexSettings.getSkipIgnoredSourceRead() == false) {
+            fieldsToLoadForSyntheticSource.add(NAME);
+        }
+
+        return fieldsToLoadForSyntheticSource;
+    }
+
+    @Override
+    protected SyntheticSourceSupport syntheticSourceSupport() {
+        // This loader controls if this field is loaded in scope of synthetic source constructions.
+        // In rare cases decoding values stored in this field can fail leading to entire source
+        // not being available.
+        // We would like to have an option to lose some values in synthetic source
+        // but have search not fail.
+        return new SyntheticSourceSupport.Native(new SourceLoader.SyntheticFieldLoader() {
+            @Override
+            public Stream<Map.Entry<String, StoredFieldLoader>> storedFieldLoaders() {
+                if (indexSettings.getSkipIgnoredSourceRead()) {
+                    return Stream.empty();
+                }
+
+                // Values are handled in `SourceLoader`.
+                return Stream.of(Map.entry(NAME, (v) -> {}));
+            }
+
+            @Override
+            public DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf) throws IOException {
+                return null;
+            }
+
+            @Override
+            public boolean hasValue() {
+                return false;
+            }
+
+            @Override
+            public void write(XContentBuilder b) throws IOException {
+
+            }
+
+            @Override
+            public String fieldName() {
+                // Does not really matter.
+                return NAME;
+            }
+
+            @Override
+            public void reset() {
+
+            }
+        });
+    }
+
     public record MappedNameValue(NameValue nameValue, XContentType type, Map<String, Object> map) {}
 
     /**

server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java

@@ -19,6 +19,7 @@ import org.apache.lucene.util.BitSet;
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader;
@@ -48,11 +49,18 @@ public class NestedObjectMapper extends ObjectMapper {
         private Explicit<Boolean> includeInParent = Explicit.IMPLICIT_FALSE;
         private final IndexVersion indexCreatedVersion;
         private final Function<Query, BitSetProducer> bitSetProducer;
+        private final IndexSettings indexSettings;
 
-        public Builder(String name, IndexVersion indexCreatedVersion, Function<Query, BitSetProducer> bitSetProducer) {
+        public Builder(
+            String name,
+            IndexVersion indexCreatedVersion,
+            Function<Query, BitSetProducer> bitSetProducer,
+            IndexSettings indexSettings
+        ) {
             super(name, Optional.empty());
             this.indexCreatedVersion = indexCreatedVersion;
             this.bitSetProducer = bitSetProducer;
+            this.indexSettings = indexSettings;
         }
 
         Builder includeInRoot(boolean includeInRoot) {
@@ -113,7 +121,8 @@ public class NestedObjectMapper extends ObjectMapper {
                 parentTypeFilter,
                 nestedTypePath,
                 nestedTypeFilter,
-                bitSetProducer
+                bitSetProducer,
+                indexSettings
             );
         }
     }
@@ -128,7 +137,8 @@ public class NestedObjectMapper extends ObjectMapper {
             NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder(
                 name,
                 parserContext.indexVersionCreated(),
-                parserContext::bitSetProducer
+                parserContext::bitSetProducer,
+                parserContext.getIndexSettings()
             );
             parseNested(name, node, builder);
             parseObjectFields(node, parserContext, builder);
@@ -195,6 +205,7 @@ public class NestedObjectMapper extends ObjectMapper {
     private final Query nestedTypeFilter;
     // Function to create a bitset for identifying parent documents
     private final Function<Query, BitSetProducer> bitsetProducer;
+    private final IndexSettings indexSettings;
 
     NestedObjectMapper(
         String name,
@@ -208,7 +219,8 @@ public class NestedObjectMapper extends ObjectMapper {
         Query parentTypeFilter,
         String nestedTypePath,
         Query nestedTypeFilter,
-        Function<Query, BitSetProducer> bitsetProducer
+        Function<Query, BitSetProducer> bitsetProducer,
+        IndexSettings indexSettings
     ) {
         super(name, fullPath, enabled, Optional.empty(), storeArraySource, dynamic, mappers);
         this.parentTypeFilter = parentTypeFilter;
@@ -217,6 +229,7 @@ public class NestedObjectMapper extends ObjectMapper {
         this.includeInParent = includeInParent;
         this.includeInRoot = includeInRoot;
         this.bitsetProducer = bitsetProducer;
+        this.indexSettings = indexSettings;
     }
 
     public Query parentTypeFilter() {
@@ -254,7 +267,7 @@ public class NestedObjectMapper extends ObjectMapper {
 
     @Override
     public ObjectMapper.Builder newBuilder(IndexVersion indexVersionCreated) {
-        NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder(leafName(), indexVersionCreated, bitsetProducer);
+        NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder(leafName(), indexVersionCreated, bitsetProducer, indexSettings);
         builder.enabled = enabled;
         builder.dynamic = dynamic;
         builder.includeInRoot = includeInRoot;
@@ -276,7 +289,8 @@ public class NestedObjectMapper extends ObjectMapper {
             parentTypeFilter,
             nestedTypePath,
             nestedTypeFilter,
-            bitsetProducer
+            bitsetProducer,
+            indexSettings
         );
     }
 
@@ -351,7 +365,8 @@ public class NestedObjectMapper extends ObjectMapper {
             parentTypeFilter,
             nestedTypePath,
             nestedTypeFilter,
-            bitsetProducer
+            bitsetProducer,
+            indexSettings
         );
     }
 
@@ -384,7 +399,9 @@ public class NestedObjectMapper extends ObjectMapper {
         }
 
         SourceLoader sourceLoader = new SourceLoader.Synthetic(() -> super.syntheticFieldLoader(mappers.values().stream(), true), NOOP);
-        var storedFieldLoader = org.elasticsearch.index.fieldvisitor.StoredFieldLoader.create(false, sourceLoader.requiredStoredFields());
+        // Some synthetic source use cases require using _ignored_source field
+        var requiredStoredFields = IgnoredSourceFieldMapper.ensureLoaded(sourceLoader.requiredStoredFields(), indexSettings);
+        var storedFieldLoader = org.elasticsearch.index.fieldvisitor.StoredFieldLoader.create(false, requiredStoredFields);
         return new NestedSyntheticFieldLoader(
             storedFieldLoader,
             sourceLoader,

server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java

@@ -118,7 +118,6 @@ public interface SourceLoader {
                 .storedFieldLoaders()
                 .map(Map.Entry::getKey)
                 .collect(Collectors.toSet());
-            this.requiredStoredFields.add(IgnoredSourceFieldMapper.NAME);
             this.metrics = metrics;
         }
 

server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java

@@ -186,9 +186,8 @@ public class FieldAliasMapperValidationTests extends ESTestCase {
     }
 
     private static NestedObjectMapper createNestedObjectMapper(String name) {
-        return new NestedObjectMapper.Builder(name, IndexVersion.current(), query -> { throw new UnsupportedOperationException(); }).build(
-            MapperBuilderContext.root(false, false)
-        );
+        return new NestedObjectMapper.Builder(name, IndexVersion.current(), query -> { throw new UnsupportedOperationException(); }, null)
+            .build(MapperBuilderContext.root(false, false));
     }
 
     private static MappingLookup createMappingLookup(

server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java

@@ -0,0 +1,144 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.apache.lucene.index.DirectoryReader;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class IgnoredSourceFieldMapperConfigurationTests extends MapperServiceTestCase {
+    public void testDisableIgnoredSourceRead() throws IOException {
+        var mapperService = mapperServiceWithCustomSettings(
+            Map.of(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING.getKey(), true),
+            b -> {
+                b.startObject("fallback_field");
+                {
+                    b.field("type", "long").field("doc_values", "false");
+                }
+                b.endObject();
+                b.startObject("disabled_object");
+                {
+                    b.field("enabled", "false");
+                    b.startObject("properties");
+                    {
+                        b.startObject("field").field("type", "keyword").endObject();
+                    }
+                    b.endObject();
+                }
+                b.endObject();
+            }
+        );
+
+        CheckedConsumer<XContentBuilder, IOException> inputDocument = b -> {
+            b.field("fallback_field", 111);
+            b.startObject("disabled_object");
+            {
+                b.field("field", "hey");
+            }
+            b.endObject();
+        };
+
+        var doc = mapperService.documentMapper().parse(source(inputDocument));
+        // Field was written.
+        assertNotNull(doc.docs().get(0).getField(IgnoredSourceFieldMapper.NAME));
+
+        String syntheticSource = syntheticSource(mapperService.documentMapper(), inputDocument);
+        // Values are not loaded.
+        assertEquals("{}", syntheticSource);
+
+        mapperService.getIndexSettings()
+            .getScopedSettings()
+            .applySettings(Settings.builder().put(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING.getKey(), false).build());
+
+        doc = mapperService.documentMapper().parse(source(inputDocument));
+        // Field was written.
+        assertNotNull(doc.docs().get(0).getField(IgnoredSourceFieldMapper.NAME));
+
+        syntheticSource = syntheticSource(mapperService.documentMapper(), inputDocument);
+        // Values are loaded.
+        assertEquals("{\"disabled_object\":{\"field\":\"hey\"},\"fallback_field\":111}", syntheticSource);
+    }
+
+    public void testDisableIgnoredSourceWrite() throws IOException {
+        var mapperService = mapperServiceWithCustomSettings(
+            Map.of(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING.getKey(), true),
+            b -> {
+                b.startObject("fallback_field");
+                {
+                    b.field("type", "long").field("doc_values", "false");
+                }
+                b.endObject();
+                b.startObject("disabled_object");
+                {
+                    b.field("enabled", "false");
+                    b.startObject("properties");
+                    {
+                        b.startObject("field").field("type", "keyword").endObject();
+                    }
+                    b.endObject();
+                }
+                b.endObject();
+            }
+        );
+
+        CheckedConsumer<XContentBuilder, IOException> inputDocument = b -> {
+            b.field("fallback_field", 111);
+            b.startObject("disabled_object");
+            {
+                b.field("field", "hey");
+            }
+            b.endObject();
+        };
+
+        var doc = mapperService.documentMapper().parse(source(inputDocument));
+        // Field is not written.
+        assertNull(doc.docs().get(0).getField(IgnoredSourceFieldMapper.NAME));
+
+        String syntheticSource = syntheticSource(mapperService.documentMapper(), inputDocument);
+        // Values are not loaded.
+        assertEquals("{}", syntheticSource);
+
+        mapperService.getIndexSettings()
+            .getScopedSettings()
+            .applySettings(Settings.builder().put(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING.getKey(), false).build());
+
+        doc = mapperService.documentMapper().parse(source(inputDocument));
+        // Field was written.
+        assertNotNull(doc.docs().get(0).getField(IgnoredSourceFieldMapper.NAME));
+
+        syntheticSource = syntheticSource(mapperService.documentMapper(), inputDocument);
+        // Values are loaded.
+        assertEquals("{\"disabled_object\":{\"field\":\"hey\"},\"fallback_field\":111}", syntheticSource);
+    }
+
+    private MapperService mapperServiceWithCustomSettings(
+        Map<String, Boolean> customSettings,
+        CheckedConsumer<XContentBuilder, IOException> mapping
+    ) throws IOException {
+        var settings = Settings.builder();
+        for (var entry : customSettings.entrySet()) {
+            settings.put(entry.getKey(), entry.getValue());
+        }
+
+        return createMapperService(settings.build(), syntheticSourceMapping(mapping));
+    }
+
+    protected void validateRoundTripReader(String syntheticSource, DirectoryReader reader, DirectoryReader roundTripReader)
+        throws IOException {
+        // Disabling this field via index settings leads to some values not being present in source and assertReaderEquals validation to
+        // fail as a result.
+        // This is expected, these settings are introduced only as a safety net when related logic blocks ingestion or search
+        // and we would rather lose some part of source but unblock the workflow.
+    }
+}

server/src/test/java/org/elasticsearch/index/mapper/NestedLookupTests.java

@@ -65,9 +65,8 @@ public class NestedLookupTests extends MapperServiceTestCase {
     }
 
     private static NestedObjectMapper buildMapper(String name) {
-        return new NestedObjectMapper.Builder(name, IndexVersion.current(), query -> { throw new UnsupportedOperationException(); }).build(
-            MapperBuilderContext.root(false, false)
-        );
+        return new NestedObjectMapper.Builder(name, IndexVersion.current(), query -> { throw new UnsupportedOperationException(); }, null)
+            .build(MapperBuilderContext.root(false, false));
     }
 
     public void testAllParentFilters() {

server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java

@@ -1505,10 +1505,10 @@ public class NestedObjectMapperTests extends MapperServiceTestCase {
     public void testMergeNested() {
         NestedObjectMapper firstMapper = new NestedObjectMapper.Builder("nested1", IndexVersion.current(), query -> {
             throw new UnsupportedOperationException();
-        }).includeInParent(true).includeInRoot(true).build(MapperBuilderContext.root(false, false));
+        }, null).includeInParent(true).includeInRoot(true).build(MapperBuilderContext.root(false, false));
         NestedObjectMapper secondMapper = new NestedObjectMapper.Builder("nested1", IndexVersion.current(), query -> {
             throw new UnsupportedOperationException();
-        }).includeInParent(false).includeInRoot(true).build(MapperBuilderContext.root(false, false));
+        }, null).includeInParent(false).includeInRoot(true).build(MapperBuilderContext.root(false, false));
 
         MapperException e = expectThrows(
             MapperException.class,
@@ -1855,7 +1855,7 @@ public class NestedObjectMapperTests extends MapperServiceTestCase {
         MergeReason mergeReason = randomFrom(MergeReason.values());
         MapperBuilderContext mapperBuilderContext = MapperBuilderContext.root(isSourceSynthetic, isDataStream, mergeReason);
         mapperBuilderContext = mapperBuilderContext.createChildContext("name", parentContainsDimensions, randomFrom(Dynamic.values()));
-        NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder("name", IndexVersion.current(), query -> null);
+        NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder("name", IndexVersion.current(), query -> null, null);
         builder.add(new Mapper.Builder("name") {
             @Override
             public Mapper build(MapperBuilderContext context) {
@@ -1876,7 +1876,7 @@ public class NestedObjectMapperTests extends MapperServiceTestCase {
         MergeReason mergeReason = randomFrom(MergeReason.values());
         {
             MapperBuilderContext mapperBuilderContext = MapperBuilderContext.root(false, false, mergeReason);
-            NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder("name", IndexVersion.current(), query -> null);
+            NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder("name", IndexVersion.current(), query -> null, null);
             NestedObjectMapper nestedObjectMapper = builder.build(mapperBuilderContext);
             MapperMergeContext mapperMergeContext = MapperMergeContext.root(isSourceSynthetic, isDataStream, mergeReason, randomLong());
             MapperMergeContext childMergeContext = nestedObjectMapper.createChildContext(mapperMergeContext, "name");
@@ -1907,7 +1907,7 @@ public class NestedObjectMapperTests extends MapperServiceTestCase {
         MergeReason mergeReason = randomFrom(MergeReason.values());
         MapperBuilderContext mapperBuilderContext = MapperBuilderContext.root(isSourceSynthetic, isDataStream, mergeReason);
         mapperBuilderContext = mapperBuilderContext.createChildContext("name", parentContainsDimensions, randomFrom(Dynamic.values()));
-        NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder("name", IndexVersion.current(), query -> null);
+        NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder("name", IndexVersion.current(), query -> null, null);
         NestedObjectMapper nestedObjectMapper = builder.build(mapperBuilderContext);
 
         MapperMergeContext mapperMergeContext = MapperMergeContext.from(mapperBuilderContext, randomLong());

server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java

@@ -913,8 +913,7 @@ public class NestedAggregatorTests extends AggregatorTestCase {
     );
 
     public static NestedObjectMapper nestedObject(String path) {
-        return new NestedObjectMapper.Builder(path, IndexVersion.current(), query -> { throw new UnsupportedOperationException(); }).build(
-            MapperBuilderContext.root(false, false)
-        );
+        return new NestedObjectMapper.Builder(path, IndexVersion.current(), query -> { throw new UnsupportedOperationException(); }, null)
+            .build(MapperBuilderContext.root(false, false));
     }
 }

server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java

@@ -197,7 +197,7 @@ public abstract class AbstractSortTestCase<T extends SortBuilder<T>> extends EST
         };
         NestedLookup nestedLookup = NestedLookup.build(List.of(new NestedObjectMapper.Builder("path", IndexVersion.current(), query -> {
             throw new UnsupportedOperationException();
-        }).build(MapperBuilderContext.root(false, false))));
+        }, null).build(MapperBuilderContext.root(false, false))));
         return new SearchExecutionContext(
             0,
             0,

x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java

@@ -16,6 +16,7 @@ import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.MapperTestUtils;
+import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.ccr.Ccr;
@@ -331,6 +332,8 @@ public class TransportResumeFollowActionTests extends ESTestCase {
         replicatedSettings.add(IndexSettings.MAX_SHINGLE_DIFF_SETTING);
         replicatedSettings.add(IndexSettings.TIME_SERIES_END_TIME);
         replicatedSettings.add(IndexSettings.PREFER_ILM_SETTING);
+        replicatedSettings.add(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING);
+        replicatedSettings.add(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING);
 
         for (Setting<?> setting : IndexScopedSettings.BUILT_IN_INDEX_SETTINGS) {
             // removed settings have no effect, they are only there for BWC

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java

@@ -18,6 +18,7 @@ import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.fielddata.FieldDataContext;
 import org.elasticsearch.index.fielddata.IndexFieldData;
@@ -80,13 +81,16 @@ import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.getOrig
 public class SemanticTextFieldMapper extends FieldMapper implements InferenceFieldMapper {
     public static final String CONTENT_TYPE = "semantic_text";
 
+    private final IndexSettings indexSettings;
+
     public static final TypeParser PARSER = new TypeParser(
-        (n, c) -> new Builder(n, c.indexVersionCreated(), c::bitSetProducer),
+        (n, c) -> new Builder(n, c.indexVersionCreated(), c::bitSetProducer, c.getIndexSettings()),
         List.of(notInMultiFields(CONTENT_TYPE), notFromDynamicTemplates(CONTENT_TYPE))
     );
 
     public static class Builder extends FieldMapper.Builder {
         private final IndexVersion indexVersionCreated;
+        private final IndexSettings indexSettings;
 
         private final Parameter<String> inferenceId = Parameter.stringParam(
             "inference_id",
@@ -113,10 +117,22 @@ public class SemanticTextFieldMapper extends FieldMapper implements InferenceFie
 
         private Function<MapperBuilderContext, ObjectMapper> inferenceFieldBuilder;
 
-        public Builder(String name, IndexVersion indexVersionCreated, Function<Query, BitSetProducer> bitSetProducer) {
+        public Builder(
+            String name,
+            IndexVersion indexVersionCreated,
+            Function<Query, BitSetProducer> bitSetProducer,
+            IndexSettings indexSettings
+        ) {
             super(name);
             this.indexVersionCreated = indexVersionCreated;
-            this.inferenceFieldBuilder = c -> createInferenceField(c, indexVersionCreated, modelSettings.get(), bitSetProducer);
+            this.indexSettings = indexSettings;
+            this.inferenceFieldBuilder = c -> createInferenceField(
+                c,
+                indexVersionCreated,
+                modelSettings.get(),
+                bitSetProducer,
+                indexSettings
+            );
         }
 
         public Builder setInferenceId(String id) {
@@ -170,13 +186,20 @@ public class SemanticTextFieldMapper extends FieldMapper implements InferenceFie
                     indexVersionCreated,
                     meta.getValue()
                 ),
-                builderParams(this, context)
+                builderParams(this, context),
+                indexSettings
             );
         }
     }
 
-    private SemanticTextFieldMapper(String simpleName, MappedFieldType mappedFieldType, BuilderParams builderParams) {
+    private SemanticTextFieldMapper(
+        String simpleName,
+        MappedFieldType mappedFieldType,
+        BuilderParams builderParams,
+        IndexSettings indexSettings
+    ) {
         super(simpleName, mappedFieldType, builderParams);
+        this.indexSettings = indexSettings;
     }
 
     @Override
@@ -188,7 +211,9 @@ public class SemanticTextFieldMapper extends FieldMapper implements InferenceFie
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new Builder(leafName(), fieldType().indexVersionCreated, fieldType().getChunksField().bitsetProducer()).init(this);
+        return new Builder(leafName(), fieldType().indexVersionCreated, fieldType().getChunksField().bitsetProducer(), indexSettings).init(
+            this
+        );
     }
 
     @Override
@@ -229,7 +254,8 @@ public class SemanticTextFieldMapper extends FieldMapper implements InferenceFie
             Builder builder = (Builder) new Builder(
                 leafName(),
                 fieldType().indexVersionCreated,
-                fieldType().getChunksField().bitsetProducer()
+                fieldType().getChunksField().bitsetProducer(),
+                indexSettings
             ).init(this);
             try {
                 mapper = builder.setModelSettings(field.inference().modelSettings())
@@ -473,19 +499,26 @@ public class SemanticTextFieldMapper extends FieldMapper implements InferenceFie
         MapperBuilderContext context,
         IndexVersion indexVersionCreated,
         @Nullable SemanticTextField.ModelSettings modelSettings,
-        Function<Query, BitSetProducer> bitSetProducer
+        Function<Query, BitSetProducer> bitSetProducer,
+        IndexSettings indexSettings
     ) {
         return new ObjectMapper.Builder(INFERENCE_FIELD, Optional.of(ObjectMapper.Subobjects.ENABLED)).dynamic(ObjectMapper.Dynamic.FALSE)
-            .add(createChunksField(indexVersionCreated, modelSettings, bitSetProducer))
+            .add(createChunksField(indexVersionCreated, modelSettings, bitSetProducer, indexSettings))
             .build(context);
     }
 
     private static NestedObjectMapper.Builder createChunksField(
         IndexVersion indexVersionCreated,
         @Nullable SemanticTextField.ModelSettings modelSettings,
-        Function<Query, BitSetProducer> bitSetProducer
+        Function<Query, BitSetProducer> bitSetProducer,
+        IndexSettings indexSettings
     ) {
-        NestedObjectMapper.Builder chunksField = new NestedObjectMapper.Builder(CHUNKS_FIELD, indexVersionCreated, bitSetProducer);
+        NestedObjectMapper.Builder chunksField = new NestedObjectMapper.Builder(
+            CHUNKS_FIELD,
+            indexVersionCreated,
+            bitSetProducer,
+            indexSettings
+        );
         chunksField.dynamic(ObjectMapper.Dynamic.FALSE);
         KeywordFieldMapper.Builder chunkTextField = new KeywordFieldMapper.Builder(CHUNKED_TEXT_FIELD, indexVersionCreated).indexed(false)
             .docValues(false);