Add Optional Source Filtering to Source Loaders (#113827) (#118459)

This change introduces optional source filtering directly within source loaders (both synthetic and stored).
The main benefit is seen in synthetic source loaders, as synthetic fields are stored independently.
By filtering while loading the synthetic source, generating the source becomes linear in the number of fields that match the filter.

This update also modifies the get document API to apply source filters earlier—directly through the source loader.
The search API, however, is not affected in this change, since the loaded source is still used by other features (e.g., highlighting, fields, nested hits),
and source filtering is always applied as the final step.
A follow-up will be required to ensure careful handling of all search-related scenarios.

benchmarks/src/main/java/org/elasticsearch/benchmark/search/fetch/subphase/FetchSourcePhaseBenchmark.java

@@ -63,7 +63,7 @@ public class FetchSourcePhaseBenchmark {
         );
         includesSet = Set.of(fetchContext.includes());
         excludesSet = Set.of(fetchContext.excludes());
-        parserConfig = XContentParserConfiguration.EMPTY.withFiltering(includesSet, excludesSet, false);
+        parserConfig = XContentParserConfiguration.EMPTY.withFiltering(null, includesSet, excludesSet, false);
     }
 
     private BytesReference read300BytesExample() throws IOException {

benchmarks/src/main/java/org/elasticsearch/benchmark/xcontent/FilterContentBenchmark.java

@@ -170,7 +170,7 @@ public class FilterContentBenchmark {
             includes = null;
             excludes = filters;
         }
-        return XContentParserConfiguration.EMPTY.withFiltering(includes, excludes, matchDotsInFieldNames);
+        return XContentParserConfiguration.EMPTY.withFiltering(null, includes, excludes, matchDotsInFieldNames);
     }
 
     private BytesReference filter(XContentParserConfiguration contentParserConfiguration) throws IOException {

docs/changelog/113827.yaml

@@ -0,0 +1,5 @@
+pr: 113827
+summary: Add Optional Source Filtering to Source Loaders
+area: Mapping
+type: enhancement
+issues: []

libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/XContentParserConfigurationImpl.java

@@ -19,6 +19,8 @@ import org.elasticsearch.xcontent.XContentParserConfiguration;
 import org.elasticsearch.xcontent.provider.filtering.FilterPathBasedFilter;
 import org.elasticsearch.xcontent.support.filtering.FilterPath;
 
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Set;
 
 public class XContentParserConfigurationImpl implements XContentParserConfiguration {
@@ -106,12 +108,41 @@ public class XContentParserConfigurationImpl implements XContentParserConfigurat
         Set<String> excludeStrings,
         boolean filtersMatchFieldNamesWithDots
     ) {
+        return withFiltering(null, includeStrings, excludeStrings, filtersMatchFieldNamesWithDots);
+    }
+
+    public XContentParserConfiguration withFiltering(
+        String prefixPath,
+        Set<String> includeStrings,
+        Set<String> excludeStrings,
+        boolean filtersMatchFieldNamesWithDots
+    ) {
+        FilterPath[] includePaths = FilterPath.compile(includeStrings);
+        FilterPath[] excludePaths = FilterPath.compile(excludeStrings);
+
+        if (prefixPath != null) {
+            if (includePaths != null) {
+                List<FilterPath> includeFilters = new ArrayList<>();
+                for (var incl : includePaths) {
+                    incl.matches(prefixPath, includeFilters, true);
+                }
+                includePaths = includeFilters.isEmpty() ? null : includeFilters.toArray(FilterPath[]::new);
+            }
+
+            if (excludePaths != null) {
+                List<FilterPath> excludeFilters = new ArrayList<>();
+                for (var excl : excludePaths) {
+                    excl.matches(prefixPath, excludeFilters, true);
+                }
+                excludePaths = excludeFilters.isEmpty() ? null : excludeFilters.toArray(FilterPath[]::new);
+            }
+        }
         return new XContentParserConfigurationImpl(
             registry,
             deprecationHandler,
             restApiVersion,
-            FilterPath.compile(includeStrings),
-            FilterPath.compile(excludeStrings),
+            includePaths,
+            excludePaths,
             filtersMatchFieldNamesWithDots
         );
     }

libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentParserConfiguration.java

@@ -49,10 +49,27 @@ public interface XContentParserConfiguration {
 
     RestApiVersion restApiVersion();
 
+    // TODO: Remove when serverless uses the new API
+    XContentParserConfiguration withFiltering(
+        Set<String> includeStrings,
+        Set<String> excludeStrings,
+        boolean filtersMatchFieldNamesWithDots
+    );
+
     /**
      * Replace the configured filtering.
+     *
+     * @param prefixPath                    The path to be prepended to each sub-path before applying the include/exclude rules.
+     *                                      Specify {@code null} if parsing starts from the root.
+     * @param includeStrings                A set of strings representing paths to include during filtering.
+     *                                      If specified, only these paths will be included in parsing.
+     * @param excludeStrings                A set of strings representing paths to exclude during filtering.
+     *                                      If specified, these paths will be excluded from parsing.
+     * @param filtersMatchFieldNamesWithDots Indicates whether filters should match field names containing dots ('.')
+     *                                      as part of the field name.
      */
     XContentParserConfiguration withFiltering(
+        String prefixPath,
         Set<String> includeStrings,
         Set<String> excludeStrings,
         boolean filtersMatchFieldNamesWithDots

libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java

@@ -22,6 +22,7 @@ import org.elasticsearch.xcontent.XContentType;
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.Set;
 import java.util.stream.IntStream;
 
@@ -332,6 +333,24 @@ public abstract class AbstractXContentFilteringTestCase extends AbstractFilterin
     private void testFilter(Builder expected, Builder sample, Set<String> includes, Set<String> excludes, boolean matchFieldNamesWithDots)
         throws IOException {
         assertFilterResult(expected.apply(createBuilder()), filter(sample, includes, excludes, matchFieldNamesWithDots));
+
+        String rootPrefix = "root.path.random";
+        if (includes != null) {
+            Set<String> rootIncludes = new HashSet<>();
+            for (var incl : includes) {
+                rootIncludes.add(rootPrefix + (randomBoolean() ? "." : "*.") + incl);
+            }
+            includes = rootIncludes;
+        }
+
+        if (excludes != null) {
+            Set<String> rootExcludes = new HashSet<>();
+            for (var excl : excludes) {
+                rootExcludes.add(rootPrefix + (randomBoolean() ? "." : "*.") + excl);
+            }
+            excludes = rootExcludes;
+        }
+        assertFilterResult(expected.apply(createBuilder()), filterSub(sample, rootPrefix, includes, excludes, matchFieldNamesWithDots));
     }
 
     public void testArrayWithEmptyObjectInInclude() throws IOException {
@@ -413,21 +432,36 @@ public abstract class AbstractXContentFilteringTestCase extends AbstractFilterin
             && matchFieldNamesWithDots == false) {
             return filterOnBuilder(sample, includes, excludes);
         }
-        return filterOnParser(sample, includes, excludes, matchFieldNamesWithDots);
+        return filterOnParser(sample, null, includes, excludes, matchFieldNamesWithDots);
+    }
+
+    private XContentBuilder filterSub(
+        Builder sample,
+        String root,
+        Set<String> includes,
+        Set<String> excludes,
+        boolean matchFieldNamesWithDots
+    ) throws IOException {
+        return filterOnParser(sample, root, includes, excludes, matchFieldNamesWithDots);
     }
 
     private XContentBuilder filterOnBuilder(Builder sample, Set<String> includes, Set<String> excludes) throws IOException {
         return sample.apply(XContentBuilder.builder(getXContentType(), includes, excludes));
     }
 
-    private XContentBuilder filterOnParser(Builder sample, Set<String> includes, Set<String> excludes, boolean matchFieldNamesWithDots)
-        throws IOException {
+    private XContentBuilder filterOnParser(
+        Builder sample,
+        String rootPath,
+        Set<String> includes,
+        Set<String> excludes,
+        boolean matchFieldNamesWithDots
+    ) throws IOException {
         try (XContentBuilder builtSample = sample.apply(createBuilder())) {
             BytesReference sampleBytes = BytesReference.bytes(builtSample);
             try (
                 XContentParser parser = getXContentType().xContent()
                     .createParser(
-                        XContentParserConfiguration.EMPTY.withFiltering(includes, excludes, matchFieldNamesWithDots),
+                        XContentParserConfiguration.EMPTY.withFiltering(rootPath, includes, excludes, matchFieldNamesWithDots),
                         sampleBytes.streamInput()
                     )
             ) {

server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java

@@ -36,6 +36,7 @@ import org.elasticsearch.script.UpdateCtxMap;
 import org.elasticsearch.script.UpdateScript;
 import org.elasticsearch.script.UpsertCtxMap;
 import org.elasticsearch.search.lookup.Source;
+import org.elasticsearch.search.lookup.SourceFilter;
 import org.elasticsearch.xcontent.XContentType;
 
 import java.io.IOException;
@@ -344,8 +345,9 @@ public class UpdateHelper {
             return null;
         }
         BytesReference sourceFilteredAsBytes = sourceAsBytes;
-        if (request.fetchSource().hasFilter()) {
-            sourceFilteredAsBytes = Source.fromMap(source, sourceContentType).filter(request.fetchSource().filter()).internalSourceRef();
+        SourceFilter sourceFilter = request.fetchSource().filter();
+        if (sourceFilter != null) {
+            sourceFilteredAsBytes = Source.fromMap(source, sourceContentType).filter(sourceFilter).internalSourceRef();
         }
 
         // TODO when using delete/none, we can still return the source as bytes by generating it (using the sourceContentType)

server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java

@@ -1361,6 +1361,7 @@ public final class DataStream implements SimpleDiffable<DataStream>, ToXContentO
     }
 
     public static final XContentParserConfiguration TS_EXTRACT_CONFIG = XContentParserConfiguration.EMPTY.withFiltering(
+        null,
         Set.of(TIMESTAMP_FIELD_NAME),
         null,
         false

server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java

@@ -273,7 +273,7 @@ public abstract class IndexRouting {
             trackTimeSeriesRoutingHash = metadata.getCreationVersion().onOrAfter(IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID);
             List<String> routingPaths = metadata.getRoutingPaths();
             isRoutingPath = Regex.simpleMatcher(routingPaths.toArray(String[]::new));
-            this.parserConfig = XContentParserConfiguration.EMPTY.withFiltering(Set.copyOf(routingPaths), null, true);
+            this.parserConfig = XContentParserConfiguration.EMPTY.withFiltering(null, Set.copyOf(routingPaths), null, true);
         }
 
         public boolean matchesField(String fieldName) {

server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java

@@ -191,7 +191,7 @@ public class XContentHelper {
     ) throws ElasticsearchParseException {
         XContentParserConfiguration config = XContentParserConfiguration.EMPTY;
         if (include != null || exclude != null) {
-            config = config.withFiltering(include, exclude, false);
+            config = config.withFiltering(null, include, exclude, false);
         }
         return parseToType(ordered ? XContentParser::mapOrdered : XContentParser::map, bytes, xContentType, config);
     }
@@ -266,7 +266,10 @@ public class XContentHelper {
         @Nullable Set<String> exclude
     ) throws ElasticsearchParseException {
         try (
-            XContentParser parser = xContent.createParser(XContentParserConfiguration.EMPTY.withFiltering(include, exclude, false), input)
+            XContentParser parser = xContent.createParser(
+                XContentParserConfiguration.EMPTY.withFiltering(null, include, exclude, false),
+                input
+            )
         ) {
             return ordered ? parser.mapOrdered() : parser.map();
         } catch (IOException e) {
@@ -301,7 +304,7 @@ public class XContentHelper {
     ) throws ElasticsearchParseException {
         try (
             XContentParser parser = xContent.createParser(
-                XContentParserConfiguration.EMPTY.withFiltering(include, exclude, false),
+                XContentParserConfiguration.EMPTY.withFiltering(null, include, exclude, false),
                 bytes,
                 offset,
                 length

server/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java

@@ -274,24 +274,8 @@ public class XContentMapValues {
      */
     public static Function<Map<String, Object>, Map<String, Object>> filter(String[] includes, String[] excludes) {
         CharacterRunAutomaton matchAllAutomaton = new CharacterRunAutomaton(Automata.makeAnyString());
-
-        CharacterRunAutomaton include;
-        if (includes == null || includes.length == 0) {
-            include = matchAllAutomaton;
-        } else {
-            Automaton includeA = Regex.simpleMatchToAutomaton(includes);
-            includeA = makeMatchDotsInFieldNames(includeA);
-            include = new CharacterRunAutomaton(includeA, MAX_DETERMINIZED_STATES);
-        }
-
-        Automaton excludeA;
-        if (excludes == null || excludes.length == 0) {
-            excludeA = Automata.makeEmpty();
-        } else {
-            excludeA = Regex.simpleMatchToAutomaton(excludes);
-            excludeA = makeMatchDotsInFieldNames(excludeA);
-        }
-        CharacterRunAutomaton exclude = new CharacterRunAutomaton(excludeA, MAX_DETERMINIZED_STATES);
+        CharacterRunAutomaton include = compileAutomaton(includes, matchAllAutomaton);
+        CharacterRunAutomaton exclude = compileAutomaton(excludes, new CharacterRunAutomaton(Automata.makeEmpty()));
 
         // NOTE: We cannot use Operations.minus because of the special case that
         // we want all sub properties to match as soon as an object matches
@@ -299,6 +283,15 @@ public class XContentMapValues {
         return (map) -> filter(map, include, 0, exclude, 0, matchAllAutomaton);
     }
 
+    public static CharacterRunAutomaton compileAutomaton(String[] patterns, CharacterRunAutomaton defaultValue) {
+        if (patterns == null || patterns.length == 0) {
+            return defaultValue;
+        }
+        var aut = Regex.simpleMatchToAutomaton(patterns);
+        aut = makeMatchDotsInFieldNames(aut);
+        return new CharacterRunAutomaton(aut, MAX_DETERMINIZED_STATES);
+    }
+
     /** Make matches on objects also match dots in field names.
      *  For instance, if the original simple regex is `foo`, this will translate
      *  it into `foo` OR `foo.*`. */

server/src/main/java/org/elasticsearch/index/engine/LuceneSyntheticSourceChangesSnapshot.java

@@ -80,7 +80,7 @@ public class LuceneSyntheticSourceChangesSnapshot extends SearchBasedChangesSnap
         assert mappingLookup.isSourceSynthetic();
         // ensure we can buffer at least one document
         this.maxMemorySizeInBytes = maxMemorySizeInBytes > 0 ? maxMemorySizeInBytes : 1;
-        this.sourceLoader = mappingLookup.newSourceLoader(SourceFieldMetrics.NOOP);
+        this.sourceLoader = mappingLookup.newSourceLoader(null, SourceFieldMetrics.NOOP);
         Set<String> storedFields = sourceLoader.requiredStoredFields();
         assert mappingLookup.isSourceSynthetic() : "synthetic source must be enabled for proper functionality.";
         this.storedFieldLoader = StoredFieldLoader.create(false, storedFields);

server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java

@@ -53,17 +53,24 @@ public abstract class StoredFieldLoader {
         return create(spec.requiresSource(), spec.requiredStoredFields());
     }
 
+    public static StoredFieldLoader create(boolean loadSource, Set<String> fields) {
+        return create(loadSource, fields, false);
+    }
+
     /**
      * Creates a new StoredFieldLoader
-     * @param loadSource should this loader load the _source field
-     * @param fields     a set of additional fields the loader should load
+     *
+     * @param loadSource           indicates whether this loader should load the {@code _source} field.
+     * @param fields               a set of additional fields that the loader should load.
+     * @param forceSequentialReader if {@code true}, forces the use of a sequential leaf reader;
+     *                              otherwise, uses the heuristic defined in {@link StoredFieldLoader#reader(LeafReaderContext, int[])}.
      */
-    public static StoredFieldLoader create(boolean loadSource, Set<String> fields) {
+    public static StoredFieldLoader create(boolean loadSource, Set<String> fields, boolean forceSequentialReader) {
         List<String> fieldsToLoad = fieldsToLoad(loadSource, fields);
         return new StoredFieldLoader() {
             @Override
             public LeafStoredFieldLoader getLoader(LeafReaderContext ctx, int[] docs) throws IOException {
-                return new ReaderStoredFieldLoader(reader(ctx, docs), loadSource, fields);
+                return new ReaderStoredFieldLoader(forceSequentialReader ? sequentialReader(ctx) : reader(ctx, docs), loadSource, fields);
             }
 
             @Override

server/src/main/java/org/elasticsearch/index/get/ShardGetService.java

@@ -306,9 +306,14 @@ public final class ShardGetService extends AbstractIndexShardComponent {
         Map<String, DocumentField> documentFields = null;
         Map<String, DocumentField> metadataFields = null;
         DocIdAndVersion docIdAndVersion = get.docIdAndVersion();
+        var sourceFilter = fetchSourceContext.filter();
         SourceLoader loader = forceSyntheticSource
-            ? new SourceLoader.Synthetic(mappingLookup.getMapping()::syntheticFieldLoader, mapperMetrics.sourceFieldMetrics())
-            : mappingLookup.newSourceLoader(mapperMetrics.sourceFieldMetrics());
+            ? new SourceLoader.Synthetic(
+                sourceFilter,
+                () -> mappingLookup.getMapping().syntheticFieldLoader(sourceFilter),
+                mapperMetrics.sourceFieldMetrics()
+            )
+            : mappingLookup.newSourceLoader(fetchSourceContext.filter(), mapperMetrics.sourceFieldMetrics());
         StoredFieldLoader storedFieldLoader = buildStoredFieldLoader(storedFields, fetchSourceContext, loader);
         LeafStoredFieldLoader leafStoredFieldLoader = storedFieldLoader.getLoader(docIdAndVersion.reader.getContext(), null);
         try {
@@ -367,10 +372,6 @@ public final class ShardGetService extends AbstractIndexShardComponent {
         if (mapperService.mappingLookup().isSourceEnabled() && fetchSourceContext.fetchSource()) {
             Source source = loader.leaf(docIdAndVersion.reader, new int[] { docIdAndVersion.docId })
                 .source(leafStoredFieldLoader, docIdAndVersion.docId);
-
-            if (fetchSourceContext.hasFilter()) {
-                source = source.filter(fetchSourceContext.filter());
-            }
             sourceBytes = source.internalSourceRef();
         }
 

server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java

@@ -155,7 +155,7 @@ public class DocumentMapper {
          * with the source loading strategy declared on the source field mapper.
          */
         try {
-            sourceMapper().newSourceLoader(mapping(), mapperMetrics.sourceFieldMetrics());
+            mappingLookup.newSourceLoader(null, mapperMetrics.sourceFieldMetrics());
         } catch (IllegalArgumentException e) {
             mapperMetrics.sourceFieldMetrics().recordSyntheticSourceIncompatibleMapping();
             throw e;

server/src/main/java/org/elasticsearch/index/mapper/FieldAliasMapper.java

@@ -156,9 +156,4 @@ public final class FieldAliasMapper extends Mapper {
             return new FieldAliasMapper(leafName(), fullName, path);
         }
     }
-
-    @Override
-    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
-        return SourceLoader.SyntheticFieldLoader.NOTHING;
-    }
 }

server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java

@@ -31,6 +31,7 @@ import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.lookup.SearchLookup;
+import org.elasticsearch.search.lookup.SourceFilter;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.ToXContentFragment;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -484,7 +485,7 @@ public abstract class FieldMapper extends Mapper {
     /**
      * Returns synthetic field loader for the mapper.
      * If mapper does not support synthetic source, it is handled using generic implementation
-     * in {@link DocumentParser#parseObjectOrField} and {@link ObjectMapper#syntheticFieldLoader()}.
+     * in {@link DocumentParser#parseObjectOrField} and {@link ObjectMapper#syntheticFieldLoader(SourceFilter)}.
      * <br>
      *
      * This method is final in order to support common use cases like fallback synthetic source.
@@ -492,7 +493,6 @@ public abstract class FieldMapper extends Mapper {
      *
      * @return implementation of {@link SourceLoader.SyntheticFieldLoader}
      */
-    @Override
     public final SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
         if (hasScript()) {
             return SourceLoader.SyntheticFieldLoader.NOTHING;

server/src/main/java/org/elasticsearch/index/mapper/Mapper.java

@@ -172,18 +172,6 @@ public abstract class Mapper implements ToXContentFragment, Iterable<Mapper> {
      */
     public abstract void validate(MappingLookup mappers);
 
-    /**
-     * Create a {@link SourceLoader.SyntheticFieldLoader} to populate synthetic source.
-     *
-     * @throws IllegalArgumentException if the field is configured in a way that doesn't
-     *         support synthetic source. This translates nicely into a 400 error when
-     *         users configure synthetic source in the mapping without configuring all
-     *         fields properly.
-     */
-    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
-        throw new IllegalArgumentException("field [" + fullPath() + "] of type [" + typeName() + "] doesn't support synthetic source");
-    }
-
     @Override
     public String toString() {
         return Strings.toString(this);

server/src/main/java/org/elasticsearch/index/mapper/Mapping.java

@@ -13,7 +13,9 @@ import org.elasticsearch.ElasticsearchGenerationException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
+import org.elasticsearch.search.lookup.SourceFilter;
 import org.elasticsearch.xcontent.ToXContentFragment;
 import org.elasticsearch.xcontent.XContentBuilder;
 
@@ -22,6 +24,7 @@ import java.util.Arrays;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 /**
@@ -126,9 +129,9 @@ public final class Mapping implements ToXContentFragment {
         return sfm != null && sfm.isSynthetic();
     }
 
-    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
-        var stream = Stream.concat(Stream.of(metadataMappers), root.mappers.values().stream());
-        return root.syntheticFieldLoader(stream);
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader(@Nullable SourceFilter filter) {
+        var mappers = Stream.concat(Stream.of(metadataMappers), root.mappers.values().stream()).collect(Collectors.toList());
+        return root.syntheticFieldLoader(filter, mappers, false);
     }
 
     /**

server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java

@@ -12,10 +12,12 @@ package org.elasticsearch.index.mapper;
 import org.apache.lucene.codecs.PostingsFormat;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.InferenceFieldMetadata;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.inference.InferenceService;
+import org.elasticsearch.search.lookup.SourceFilter;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -497,9 +499,11 @@ public final class MappingLookup {
     /**
      * Build something to load source {@code _source}.
      */
-    public SourceLoader newSourceLoader(SourceFieldMetrics metrics) {
-        SourceFieldMapper sfm = mapping.getMetadataMapperByClass(SourceFieldMapper.class);
-        return sfm == null ? SourceLoader.FROM_STORED_SOURCE : sfm.newSourceLoader(mapping, metrics);
+    public SourceLoader newSourceLoader(@Nullable SourceFilter filter, SourceFieldMetrics metrics) {
+        if (isSourceSynthetic()) {
+            return new SourceLoader.Synthetic(filter, () -> mapping.syntheticFieldLoader(filter), metrics);
+        }
+        return filter == null ? SourceLoader.FROM_STORED_SOURCE : new SourceLoader.Stored(filter);
     }
 
     /**

server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java

@@ -23,10 +23,12 @@ import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader;
+import org.elasticsearch.search.lookup.SourceFilter;
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -403,16 +405,18 @@ public class NestedObjectMapper extends ObjectMapper {
     }
 
     @Override
-    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+    SourceLoader.SyntheticFieldLoader syntheticFieldLoader(SourceFilter filter, Collection<Mapper> mappers, boolean isFragment) {
+        // IgnoredSourceFieldMapper integration takes care of writing the source for nested objects that enabled store_array_source.
         if (sourceKeepMode.orElse(SourceKeepMode.NONE) == SourceKeepMode.ALL) {
             // IgnoredSourceFieldMapper integration takes care of writing the source for the nested object.
             return SourceLoader.SyntheticFieldLoader.NOTHING;
         }
 
-        SourceLoader sourceLoader = new SourceLoader.Synthetic(() -> super.syntheticFieldLoader(mappers.values().stream(), true), NOOP);
+        SourceLoader sourceLoader = new SourceLoader.Synthetic(filter, () -> super.syntheticFieldLoader(filter, mappers, true), NOOP);
         // Some synthetic source use cases require using _ignored_source field
         var requiredStoredFields = IgnoredSourceFieldMapper.ensureLoaded(sourceLoader.requiredStoredFields(), indexSettings);
-        var storedFieldLoader = org.elasticsearch.index.fieldvisitor.StoredFieldLoader.create(false, requiredStoredFields);
+        // force sequential access since nested fields are indexed per block
+        var storedFieldLoader = org.elasticsearch.index.fieldvisitor.StoredFieldLoader.create(false, requiredStoredFields, true);
         return new NestedSyntheticFieldLoader(
             storedFieldLoader,
             sourceLoader,
@@ -504,5 +508,10 @@ public class NestedObjectMapper extends ObjectMapper {
         public String fieldName() {
             return NestedObjectMapper.this.fullPath();
         }
+
+        @Override
+        public void reset() {
+            children.clear();
+        }
     }
 }

server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java

@@ -24,8 +24,10 @@ import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
+import org.elasticsearch.search.lookup.SourceFilter;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -39,6 +41,7 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.Set;
 import java.util.TreeMap;
 import java.util.stream.Stream;
 
@@ -888,24 +891,40 @@ public class ObjectMapper extends Mapper {
         return null;
     }
 
-    protected SourceLoader.SyntheticFieldLoader syntheticFieldLoader(Stream<Mapper> mappers, boolean isFragment) {
-        var fields = mappers.sorted(Comparator.comparing(Mapper::fullPath))
-            .map(Mapper::syntheticFieldLoader)
+    SourceLoader.SyntheticFieldLoader syntheticFieldLoader(SourceFilter filter, Collection<Mapper> mappers, boolean isFragment) {
+        var fields = mappers.stream()
+            .sorted(Comparator.comparing(Mapper::fullPath))
+            .map(m -> innerSyntheticFieldLoader(filter, m))
             .filter(l -> l != SourceLoader.SyntheticFieldLoader.NOTHING)
             .toList();
-        return new SyntheticSourceFieldLoader(fields, isFragment);
+        return new SyntheticSourceFieldLoader(filter, fields, isFragment);
     }
 
-    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader(Stream<Mapper> mappers) {
-        return syntheticFieldLoader(mappers, false);
+    final SourceLoader.SyntheticFieldLoader syntheticFieldLoader(@Nullable SourceFilter filter) {
+        return syntheticFieldLoader(filter, mappers.values(), false);
     }
 
-    @Override
-    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
-        return syntheticFieldLoader(mappers.values().stream());
+    private SourceLoader.SyntheticFieldLoader innerSyntheticFieldLoader(SourceFilter filter, Mapper mapper) {
+        if (mapper instanceof MetadataFieldMapper metaMapper) {
+            return metaMapper.syntheticFieldLoader();
+        }
+        if (filter != null && filter.isPathFiltered(mapper.fullPath(), mapper instanceof ObjectMapper)) {
+            return SourceLoader.SyntheticFieldLoader.NOTHING;
+        }
+
+        if (mapper instanceof ObjectMapper objectMapper) {
+            return objectMapper.syntheticFieldLoader(filter);
+        }
+
+        if (mapper instanceof FieldMapper fieldMapper) {
+            return fieldMapper.syntheticFieldLoader();
+        }
+        return SourceLoader.SyntheticFieldLoader.NOTHING;
     }
 
     private class SyntheticSourceFieldLoader implements SourceLoader.SyntheticFieldLoader {
+        private final SourceFilter filter;
+        private final XContentParserConfiguration parserConfig;
         private final List<SourceLoader.SyntheticFieldLoader> fields;
         private final boolean isFragment;
 
@@ -921,9 +940,19 @@ public class ObjectMapper extends Mapper {
         // Use an ordered map between field names and writers to order writing by field name.
         private TreeMap<String, FieldWriter> currentWriters;
 
-        private SyntheticSourceFieldLoader(List<SourceLoader.SyntheticFieldLoader> fields, boolean isFragment) {
+        private SyntheticSourceFieldLoader(SourceFilter filter, List<SourceLoader.SyntheticFieldLoader> fields, boolean isFragment) {
             this.fields = fields;
             this.isFragment = isFragment;
+            this.filter = filter;
+            String fullPath = ObjectMapper.this.isRoot() ? null : fullPath();
+            this.parserConfig = filter == null
+                ? XContentParserConfiguration.EMPTY
+                : XContentParserConfiguration.EMPTY.withFiltering(
+                    fullPath,
+                    filter.getIncludes() != null ? Set.of(filter.getIncludes()) : null,
+                    filter.getExcludes() != null ? Set.of(filter.getExcludes()) : null,
+                    true
+                );
         }
 
         @Override
@@ -994,7 +1023,7 @@ public class ObjectMapper extends Mapper {
 
                     var existing = currentWriters.get(value.name());
                     if (existing == null) {
-                        currentWriters.put(value.name(), new FieldWriter.IgnoredSource(value));
+                        currentWriters.put(value.name(), new FieldWriter.IgnoredSource(filter, value));
                     } else if (existing instanceof FieldWriter.IgnoredSource isw) {
                         isw.mergeWith(value);
                     }
@@ -1031,7 +1060,10 @@ public class ObjectMapper extends Mapper {
                 // If the root object mapper is disabled, it is expected to contain
                 // the source encapsulated within a single ignored source value.
                 assert ignoredValues.size() == 1 : ignoredValues.size();
-                XContentDataHelper.decodeAndWrite(b, ignoredValues.get(0).value());
+                var value = ignoredValues.get(0).value();
+                var type = XContentDataHelper.decodeType(value);
+                assert type.isPresent();
+                XContentDataHelper.decodeAndWriteXContent(parserConfig, b, type.get(), ignoredValues.get(0).value());
                 softReset();
                 return;
             }
@@ -1109,11 +1141,20 @@ public class ObjectMapper extends Mapper {
             }
 
             class IgnoredSource implements FieldWriter {
+                private final XContentParserConfiguration parserConfig;
                 private final String fieldName;
                 private final String leafName;
                 private final List<BytesRef> encodedValues;
 
-                IgnoredSource(IgnoredSourceFieldMapper.NameValue initialValue) {
+                IgnoredSource(SourceFilter filter, IgnoredSourceFieldMapper.NameValue initialValue) {
+                    parserConfig = filter == null
+                        ? XContentParserConfiguration.EMPTY
+                        : XContentParserConfiguration.EMPTY.withFiltering(
+                            initialValue.name(),
+                            filter.getIncludes() != null ? Set.of(filter.getIncludes()) : null,
+                            filter.getExcludes() != null ? Set.of(filter.getExcludes()) : null,
+                            true
+                        );
                     this.fieldName = initialValue.name();
                     this.leafName = initialValue.getFieldName();
                     this.encodedValues = new ArrayList<>();
@@ -1124,7 +1165,7 @@ public class ObjectMapper extends Mapper {
 
                 @Override
                 public void writeTo(XContentBuilder builder) throws IOException {
-                    XContentDataHelper.writeMerged(builder, leafName, encodedValues);
+                    XContentDataHelper.writeMerged(parserConfig, builder, leafName, encodedValues);
                 }
 
                 @Override

server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java

@@ -453,16 +453,6 @@ public class SourceFieldMapper extends MetadataFieldMapper {
         return new Builder(null, Settings.EMPTY, false, serializeMode).init(this);
     }
 
-    /**
-     * Build something to load source {@code _source}.
-     */
-    public SourceLoader newSourceLoader(Mapping mapping, SourceFieldMetrics metrics) {
-        if (mode == Mode.SYNTHETIC) {
-            return new SourceLoader.Synthetic(mapping::syntheticFieldLoader, metrics);
-        }
-        return SourceLoader.FROM_STORED_SOURCE;
-    }
-
     public boolean isSynthetic() {
         return mode == Mode.SYNTHETIC;
     }

server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java

@@ -11,9 +11,11 @@ package org.elasticsearch.index.mapper;
 
 import org.apache.lucene.index.LeafReader;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader;
 import org.elasticsearch.search.lookup.Source;
+import org.elasticsearch.search.lookup.SourceFilter;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.json.JsonXContent;
 
@@ -71,7 +73,15 @@ public interface SourceLoader {
     /**
      * Load {@code _source} from a stored field.
      */
-    SourceLoader FROM_STORED_SOURCE = new SourceLoader() {
+    SourceLoader FROM_STORED_SOURCE = new Stored(null);
+
+    class Stored implements SourceLoader {
+        final SourceFilter filter;
+
+        public Stored(@Nullable SourceFilter filter) {
+            this.filter = filter;
+        }
+
         @Override
         public boolean reordersFieldValues() {
             return false;
@@ -82,7 +92,8 @@ public interface SourceLoader {
             return new Leaf() {
                 @Override
                 public Source source(LeafStoredFieldLoader storedFields, int docId) throws IOException {
-                    return Source.fromBytes(storedFields.source());
+                    var res = Source.fromBytes(storedFields.source());
+                    return filter == null ? res : res.filter(filter);
                 }
 
                 @Override
@@ -97,28 +108,31 @@ public interface SourceLoader {
         public Set<String> requiredStoredFields() {
             return Set.of();
         }
-    };
+    }
 
     /**
      * Reconstructs {@code _source} from doc values anf stored fields.
      */
     class Synthetic implements SourceLoader {
+        private final SourceFilter filter;
         private final Supplier<SyntheticFieldLoader> syntheticFieldLoaderLeafSupplier;
         private final Set<String> requiredStoredFields;
         private final SourceFieldMetrics metrics;
 
         /**
          * Creates a {@link SourceLoader} to reconstruct {@code _source} from doc values anf stored fields.
+         * @param filter An optional filter to include/exclude fields.
          * @param fieldLoaderSupplier A supplier to create {@link SyntheticFieldLoader}, one for each leaf.
          * @param metrics Metrics for profiling.
          */
-        public Synthetic(Supplier<SyntheticFieldLoader> fieldLoaderSupplier, SourceFieldMetrics metrics) {
+        public Synthetic(@Nullable SourceFilter filter, Supplier<SyntheticFieldLoader> fieldLoaderSupplier, SourceFieldMetrics metrics) {
             this.syntheticFieldLoaderLeafSupplier = fieldLoaderSupplier;
             this.requiredStoredFields = syntheticFieldLoaderLeafSupplier.get()
                 .storedFieldLoaders()
                 .map(Map.Entry::getKey)
                 .collect(Collectors.toSet());
             this.metrics = metrics;
+            this.filter = filter;
         }
 
         @Override
@@ -134,7 +148,7 @@ public interface SourceLoader {
         @Override
         public Leaf leaf(LeafReader reader, int[] docIdsInLeaf) throws IOException {
             SyntheticFieldLoader loader = syntheticFieldLoaderLeafSupplier.get();
-            return new LeafWithMetrics(new SyntheticLeaf(loader, loader.docValuesLoader(reader, docIdsInLeaf)), metrics);
+            return new LeafWithMetrics(new SyntheticLeaf(filter, loader, loader.docValuesLoader(reader, docIdsInLeaf)), metrics);
         }
 
         private record LeafWithMetrics(Leaf leaf, SourceFieldMetrics metrics) implements Leaf {
@@ -163,11 +177,13 @@ public interface SourceLoader {
         }
 
         private static class SyntheticLeaf implements Leaf {
+            private final SourceFilter filter;
             private final SyntheticFieldLoader loader;
             private final SyntheticFieldLoader.DocValuesLoader docValuesLoader;
             private final Map<String, SyntheticFieldLoader.StoredFieldLoader> storedFieldLoaders;
 
-            private SyntheticLeaf(SyntheticFieldLoader loader, SyntheticFieldLoader.DocValuesLoader docValuesLoader) {
+            private SyntheticLeaf(SourceFilter filter, SyntheticFieldLoader loader, SyntheticFieldLoader.DocValuesLoader docValuesLoader) {
+                this.filter = filter;
                 this.loader = loader;
                 this.docValuesLoader = docValuesLoader;
                 this.storedFieldLoaders = Map.copyOf(
@@ -199,6 +215,11 @@ public interface SourceLoader {
                                 objectsWithIgnoredFields = new HashMap<>();
                             }
                             IgnoredSourceFieldMapper.NameValue nameValue = IgnoredSourceFieldMapper.decode(value);
+                            if (filter != null
+                                && filter.isPathFiltered(nameValue.name(), XContentDataHelper.isEncodedObject(nameValue.value()))) {
+                                // This path is filtered by the include/exclude rules
+                                continue;
+                            }
                             objectsWithIgnoredFields.computeIfAbsent(nameValue.getParentFieldName(), k -> new ArrayList<>()).add(nameValue);
                         }
                     }

server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java

@@ -16,6 +16,7 @@ import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.util.ByteUtils;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.CheckedFunction;
+import org.elasticsearch.core.CheckedRunnable;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
@@ -109,29 +110,53 @@ public final class XContentDataHelper {
         }
     }
 
+    /**
+     * Determines if the given {@link BytesRef}, encoded with {@link XContentDataHelper#encodeToken(XContentParser)},
+     * is an encoded object.
+     */
+    static boolean isEncodedObject(BytesRef encoded) {
+        return switch ((char) encoded.bytes[encoded.offset]) {
+            case CBOR_OBJECT_ENCODING, YAML_OBJECT_ENCODING, JSON_OBJECT_ENCODING, SMILE_OBJECT_ENCODING -> true;
+            default -> false;
+        };
+    }
+
+    static Optional<XContentType> decodeType(BytesRef encodedValue) {
+        return switch ((char) encodedValue.bytes[encodedValue.offset]) {
+            case CBOR_OBJECT_ENCODING, JSON_OBJECT_ENCODING, YAML_OBJECT_ENCODING, SMILE_OBJECT_ENCODING -> Optional.of(
+                getXContentType(encodedValue)
+            );
+            default -> Optional.empty();
+        };
+    }
+
     /**
      * Writes encoded values to provided builder. If there are multiple values they are merged into
      * a single resulting array.
      *
      * Note that this method assumes all encoded parts have values that need to be written (are not VOID encoded).
+     * @param parserConfig The configuration for the parsing of the provided {@code encodedParts}.
      * @param b destination
      * @param fieldName name of the field that is written
      * @param encodedParts subset of field data encoded using methods of this class. Can contain arrays which will be flattened.
      * @throws IOException
      */
-    static void writeMerged(XContentBuilder b, String fieldName, List<BytesRef> encodedParts) throws IOException {
+    static void writeMerged(XContentParserConfiguration parserConfig, XContentBuilder b, String fieldName, List<BytesRef> encodedParts)
+        throws IOException {
         if (encodedParts.isEmpty()) {
             return;
         }
 
-        if (encodedParts.size() == 1) {
-            b.field(fieldName);
-            XContentDataHelper.decodeAndWrite(b, encodedParts.get(0));
-            return;
-        }
-
-        b.startArray(fieldName);
+        boolean isArray = encodedParts.size() > 1;
+        // xcontent filtering can remove all values so we delay the start of the field until we have an actual value to write.
+        CheckedRunnable<IOException> startField = () -> {
+            if (isArray) {
+                b.startArray(fieldName);
+            } else {
+                b.field(fieldName);
+            }
 
+        };
         for (var encodedValue : encodedParts) {
             Optional<XContentType> encodedXContentType = switch ((char) encodedValue.bytes[encodedValue.offset]) {
                 case CBOR_OBJECT_ENCODING, JSON_OBJECT_ENCODING, YAML_OBJECT_ENCODING, SMILE_OBJECT_ENCODING -> Optional.of(
@@ -140,27 +165,33 @@ public final class XContentDataHelper {
                 default -> Optional.empty();
             };
             if (encodedXContentType.isEmpty()) {
+                if (startField != null) {
+                    // first value to write
+                    startField.run();
+                    startField = null;
+                }
                 // This is a plain value, we can just write it
                 XContentDataHelper.decodeAndWrite(b, encodedValue);
             } else {
-                // Encoded value could be an array which needs to be flattened
-                // since we are already inside an array.
+                // Encoded value could be an object or an array of objects that needs
+                // to be filtered or flattened.
                 try (
                     XContentParser parser = encodedXContentType.get()
                         .xContent()
-                        .createParser(
-                            XContentParserConfiguration.EMPTY,
-                            encodedValue.bytes,
-                            encodedValue.offset + 1,
-                            encodedValue.length - 1
-                        )
+                        .createParser(parserConfig, encodedValue.bytes, encodedValue.offset + 1, encodedValue.length - 1)
                 ) {
-                    if (parser.currentToken() == null) {
-                        parser.nextToken();
+                    if ((parser.currentToken() == null) && (parser.nextToken() == null)) {
+                        // the entire content is filtered by include/exclude rules
+                        continue;
                     }
 
-                    // It's an array, we will flatten it.
-                    if (parser.currentToken() == XContentParser.Token.START_ARRAY) {
+                    if (startField != null) {
+                        // first value to write
+                        startField.run();
+                        startField = null;
+                    }
+                    if (isArray && parser.currentToken() == XContentParser.Token.START_ARRAY) {
+                        // Encoded value is an array which needs to be flattened since we are already inside an array.
                         while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
                             b.copyCurrentStructure(parser);
                         }
@@ -171,8 +202,9 @@ public final class XContentDataHelper {
                 }
             }
         }
-
-        b.endArray();
+        if (isArray) {
+            b.endArray();
+        }
     }
 
     public static boolean isDataPresent(BytesRef encoded) {
@@ -509,10 +541,10 @@ public final class XContentDataHelper {
             @Override
             void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException {
                 switch ((char) r.bytes[r.offset]) {
-                    case CBOR_OBJECT_ENCODING -> decodeAndWriteXContent(b, XContentType.CBOR, r);
-                    case JSON_OBJECT_ENCODING -> decodeAndWriteXContent(b, XContentType.JSON, r);
-                    case SMILE_OBJECT_ENCODING -> decodeAndWriteXContent(b, XContentType.SMILE, r);
-                    case YAML_OBJECT_ENCODING -> decodeAndWriteXContent(b, XContentType.YAML, r);
+                    case CBOR_OBJECT_ENCODING -> decodeAndWriteXContent(XContentParserConfiguration.EMPTY, b, XContentType.CBOR, r);
+                    case JSON_OBJECT_ENCODING -> decodeAndWriteXContent(XContentParserConfiguration.EMPTY, b, XContentType.JSON, r);
+                    case SMILE_OBJECT_ENCODING -> decodeAndWriteXContent(XContentParserConfiguration.EMPTY, b, XContentType.SMILE, r);
+                    case YAML_OBJECT_ENCODING -> decodeAndWriteXContent(XContentParserConfiguration.EMPTY, b, XContentType.YAML, r);
                     default -> throw new IllegalArgumentException("Can't decode " + r);
                 }
             }
@@ -606,11 +638,15 @@ public final class XContentDataHelper {
             assert position == encoded.length;
             return encoded;
         }
+    }
 
-        static void decodeAndWriteXContent(XContentBuilder b, XContentType type, BytesRef r) throws IOException {
-            try (
-                XContentParser parser = type.xContent().createParser(XContentParserConfiguration.EMPTY, r.bytes, r.offset + 1, r.length - 1)
-            ) {
+    public static void decodeAndWriteXContent(XContentParserConfiguration parserConfig, XContentBuilder b, XContentType type, BytesRef r)
+        throws IOException {
+        try (XContentParser parser = type.xContent().createParser(parserConfig, r.bytes, r.offset + 1, r.length - 1)) {
+            if ((parser.currentToken() == null) && (parser.nextToken() == null)) {
+                // This can occur when all fields in a sub-object or all entries in an array of objects have been filtered out.
+                b.startObject().endObject();
+            } else {
                 b.copyCurrentStructure(parser);
             }
         }

server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java

@@ -439,9 +439,13 @@ public class SearchExecutionContext extends QueryRewriteContext {
      */
     public SourceLoader newSourceLoader(boolean forceSyntheticSource) {
         if (forceSyntheticSource) {
-            return new SourceLoader.Synthetic(mappingLookup.getMapping()::syntheticFieldLoader, mapperMetrics.sourceFieldMetrics());
+            return new SourceLoader.Synthetic(
+                null,
+                () -> mappingLookup.getMapping().syntheticFieldLoader(null),
+                mapperMetrics.sourceFieldMetrics()
+            );
         }
-        return mappingLookup.newSourceLoader(mapperMetrics.sourceFieldMetrics());
+        return mappingLookup.newSourceLoader(null, mapperMetrics.sourceFieldMetrics());
     }
 
     /**
@@ -501,7 +505,7 @@ public class SearchExecutionContext extends QueryRewriteContext {
 
     public SourceProvider createSourceProvider() {
         return isSourceSynthetic()
-            ? SourceProvider.fromSyntheticSource(mappingLookup.getMapping(), mapperMetrics.sourceFieldMetrics())
+            ? SourceProvider.fromSyntheticSource(mappingLookup.getMapping(), null, mapperMetrics.sourceFieldMetrics())
             : SourceProvider.fromStoredFields();
     }
 

server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourceContext.java

@@ -85,12 +85,15 @@ public class FetchSourceContext implements Writeable, ToXContentObject {
         return this.excludes;
     }
 
-    public boolean hasFilter() {
+    private boolean hasFilter() {
         return this.includes.length > 0 || this.excludes.length > 0;
     }
 
+    /**
+     * Returns a {@link SourceFilter} if filtering is enabled, {@code null} otherwise.
+     */
     public SourceFilter filter() {
-        return new SourceFilter(includes, excludes);
+        return hasFilter() ? new SourceFilter(includes, excludes) : null;
     }
 
     public static FetchSourceContext parseFromRestRequest(RestRequest request) {

server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java

@@ -29,7 +29,7 @@ public final class FetchSourcePhase implements FetchSubPhase {
         }
         assert fetchSourceContext.fetchSource();
         SourceFilter sourceFilter = fetchSourceContext.filter();
-        final boolean filterExcludesAll = sourceFilter.excludesAll();
+        final boolean filterExcludesAll = sourceFilter != null && sourceFilter.excludesAll();
         return new FetchSubPhaseProcessor() {
             private int fastPath;
 
@@ -47,22 +47,22 @@ public final class FetchSourcePhase implements FetchSubPhase {
             public void process(HitContext hitContext) {
                 String index = fetchContext.getIndexName();
                 if (fetchContext.getSearchExecutionContext().isSourceEnabled() == false) {
-                    if (fetchSourceContext.hasFilter()) {
+                    if (sourceFilter != null) {
                         throw new IllegalArgumentException(
                             "unable to fetch fields from _source field: _source is disabled in the mappings for index [" + index + "]"
                         );
                     }
                     return;
                 }
-                hitExecute(fetchSourceContext, hitContext);
+                hitExecute(hitContext);
             }
 
-            private void hitExecute(FetchSourceContext fetchSourceContext, HitContext hitContext) {
+            private void hitExecute(HitContext hitContext) {
                 final boolean nestedHit = hitContext.hit().getNestedIdentity() != null;
                 Source source = hitContext.source();
 
                 // If this is a parent document and there are no source filters, then add the source as-is.
-                if (nestedHit == false && fetchSourceContext.hasFilter() == false) {
+                if (nestedHit == false && sourceFilter == null) {
                     hitContext.hit().sourceRef(source.internalSourceRef());
                     fastPath++;
                     return;
@@ -73,7 +73,7 @@ public final class FetchSourcePhase implements FetchSubPhase {
                     source = Source.empty(source.sourceContentType());
                 } else {
                     // Otherwise, filter the source and add it to the hit.
-                    source = source.filter(sourceFilter);
+                    source = sourceFilter != null ? source.filter(sourceFilter) : source;
                 }
                 if (nestedHit) {
                     source = extractNested(source, hitContext.hit().getNestedIdentity());

server/src/main/java/org/elasticsearch/search/lookup/SourceFilter.java

@@ -9,6 +9,8 @@
 
 package org.elasticsearch.search.lookup;
 
+import org.apache.lucene.util.automaton.Automata;
+import org.apache.lucene.util.automaton.CharacterRunAutomaton;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
@@ -40,6 +42,8 @@ public final class SourceFilter {
     private final boolean empty;
     private final String[] includes;
     private final String[] excludes;
+    private CharacterRunAutomaton includeAut;
+    private CharacterRunAutomaton excludeAut;
 
     /**
      * Construct a new filter based on a list of includes and excludes
@@ -56,6 +60,53 @@ public final class SourceFilter {
         this.empty = CollectionUtils.isEmpty(this.includes) && CollectionUtils.isEmpty(this.excludes);
     }
 
+    public String[] getIncludes() {
+        return includes;
+    }
+
+    public String[] getExcludes() {
+        return excludes;
+    }
+
+    /**
+     * Determines whether the given full path should be filtered out.
+     *
+     * @param fullPath The full path to evaluate.
+     * @param isObject Indicates if the path represents an object.
+     * @return {@code true} if the path should be filtered out, {@code false} otherwise.
+     */
+    public boolean isPathFiltered(String fullPath, boolean isObject) {
+        final boolean included;
+        if (includes != null) {
+            if (includeAut == null) {
+                includeAut = XContentMapValues.compileAutomaton(includes, new CharacterRunAutomaton(Automata.makeAnyString()));
+            }
+            int state = step(includeAut, fullPath, 0);
+            included = state != -1 && (isObject || includeAut.isAccept(state));
+        } else {
+            included = true;
+        }
+
+        if (excludes != null) {
+            if (excludeAut == null) {
+                excludeAut = XContentMapValues.compileAutomaton(excludes, new CharacterRunAutomaton(Automata.makeEmpty()));
+            }
+            int state = step(excludeAut, fullPath, 0);
+            if (state != -1 && excludeAut.isAccept(state)) {
+                return true;
+            }
+        }
+
+        return included == false;
+    }
+
+    private static int step(CharacterRunAutomaton automaton, String key, int state) {
+        for (int i = 0; state != -1 && i < key.length(); ++i) {
+            state = automaton.step(state, key.charAt(i));
+        }
+        return state;
+    }
+
     /**
      * Filter a Source using its map representation
      */
@@ -87,6 +138,7 @@ public final class SourceFilter {
             return this::filterMap;
         }
         final XContentParserConfiguration parserConfig = XContentParserConfiguration.EMPTY.withFiltering(
+            null,
             Set.copyOf(Arrays.asList(includes)),
             Set.copyOf(Arrays.asList(excludes)),
             true

server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java

@@ -48,7 +48,7 @@ public interface SourceProvider {
      * but it is not safe to use this to access documents from the same segment across
      * multiple threads.
      */
-    static SourceProvider fromSyntheticSource(Mapping mapping, SourceFieldMetrics metrics) {
-        return new SyntheticSourceProvider(new SourceLoader.Synthetic(mapping::syntheticFieldLoader, metrics));
+    static SourceProvider fromSyntheticSource(Mapping mapping, SourceFilter filter, SourceFieldMetrics metrics) {
+        return new SyntheticSourceProvider(new SourceLoader.Synthetic(filter, () -> mapping.syntheticFieldLoader(filter), metrics));
     }
 }

server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java

@@ -98,7 +98,7 @@ public class DocCountFieldMapperTests extends MetadataMapperTestCase {
                 iw.addDocument(mapper.documentMapper().parse(source(b -> b.field("doc", doc).field(CONTENT_TYPE, c))).rootDoc());
             }
         }, reader -> {
-            SourceLoader loader = mapper.mappingLookup().newSourceLoader(SourceFieldMetrics.NOOP);
+            SourceLoader loader = mapper.mappingLookup().newSourceLoader(null, SourceFieldMetrics.NOOP);
             assertThat(loader.requiredStoredFields(), Matchers.contains("_ignored_source"));
             for (LeafReaderContext leaf : reader.leaves()) {
                 int[] docIds = IntStream.range(0, leaf.reader().maxDoc()).toArray();
@@ -130,7 +130,7 @@ public class DocCountFieldMapperTests extends MetadataMapperTestCase {
                 })).rootDoc());
             }
         }, reader -> {
-            SourceLoader loader = mapper.mappingLookup().newSourceLoader(SourceFieldMetrics.NOOP);
+            SourceLoader loader = mapper.mappingLookup().newSourceLoader(null, SourceFieldMetrics.NOOP);
             assertThat(loader.requiredStoredFields(), Matchers.contains("_ignored_source"));
             for (LeafReaderContext leaf : reader.leaves()) {
                 int[] docIds = IntStream.range(0, leaf.reader().maxDoc()).toArray();

server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java

@@ -12,6 +12,8 @@ package org.elasticsearch.index.mapper;
 import org.apache.lucene.index.DirectoryReader;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.search.lookup.SourceFilter;
 import org.elasticsearch.test.FieldMaskingReader;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.hamcrest.Matchers;
@@ -46,8 +48,15 @@ public class IgnoredSourceFieldMapperTests extends MapperServiceTestCase {
     }
 
     private String getSyntheticSourceWithFieldLimit(CheckedConsumer<XContentBuilder, IOException> build) throws IOException {
+        return getSyntheticSourceWithFieldLimit(null, build);
+    }
+
+    private String getSyntheticSourceWithFieldLimit(
+        @Nullable SourceFilter sourceFilter,
+        CheckedConsumer<XContentBuilder, IOException> build
+    ) throws IOException {
         DocumentMapper documentMapper = getDocumentMapperWithFieldLimit();
-        return syntheticSource(documentMapper, build);
+        return syntheticSource(documentMapper, sourceFilter, build);
     }
 
     private MapperService createMapperServiceWithStoredArraySource(XContentBuilder mappings) throws IOException {
@@ -62,36 +71,120 @@ public class IgnoredSourceFieldMapperTests extends MapperServiceTestCase {
     public void testIgnoredBoolean() throws IOException {
         boolean value = randomBoolean();
         assertEquals("{\"my_value\":" + value + "}", getSyntheticSourceWithFieldLimit(b -> b.field("my_value", value)));
+        assertEquals(
+            "{\"my_value\":" + value + "}",
+            getSyntheticSourceWithFieldLimit(new SourceFilter(new String[] { "my_value" }, null), b -> b.field("my_value", value))
+        );
+        assertEquals(
+            "{}",
+            getSyntheticSourceWithFieldLimit(new SourceFilter(null, new String[] { "my_value" }), b -> b.field("my_value", value))
+        );
+    }
+
+    public void testIgnoredBooleanArray() throws IOException {
+        assertEquals(
+            "{\"my_value\":[false,true,false]}",
+            getSyntheticSourceWithFieldLimit(b -> b.field("my_value", new boolean[] { false, true, false }))
+        );
+        assertEquals(
+            "{\"my_value\":[false,true,false]}",
+            getSyntheticSourceWithFieldLimit(
+                new SourceFilter(new String[] { "my_value" }, null),
+                b -> b.array("my_value", new boolean[] { false, true, false })
+            )
+        );
+        assertEquals(
+            "{}",
+            getSyntheticSourceWithFieldLimit(
+                new SourceFilter(null, new String[] { "my_value" }),
+                b -> b.field("my_value", new boolean[] { false, true, false })
+            )
+        );
+        assertEquals(
+            "{}",
+            getSyntheticSourceWithFieldLimit(
+                new SourceFilter(new String[] { "my_value.object" }, null),
+                b -> b.array("my_value", new boolean[] { false, true, false })
+            )
+        );
     }
 
     public void testIgnoredString() throws IOException {
         String value = randomAlphaOfLength(5);
         assertEquals("{\"my_value\":\"" + value + "\"}", getSyntheticSourceWithFieldLimit(b -> b.field("my_value", value)));
+        assertEquals(
+            "{\"my_value\":\"" + value + "\"}",
+            getSyntheticSourceWithFieldLimit(new SourceFilter(new String[] { "my_value" }, null), b -> b.field("my_value", value))
+        );
+        assertEquals(
+            "{}",
+            getSyntheticSourceWithFieldLimit(new SourceFilter(null, new String[] { "my_value" }), b -> b.field("my_value", value))
+        );
     }
 
     public void testIgnoredInt() throws IOException {
         int value = randomInt();
         assertEquals("{\"my_value\":" + value + "}", getSyntheticSourceWithFieldLimit(b -> b.field("my_value", value)));
+        assertEquals(
+            "{\"my_value\":" + value + "}",
+            getSyntheticSourceWithFieldLimit(new SourceFilter(new String[] { "my_value" }, null), b -> b.field("my_value", value))
+        );
+        assertEquals(
+            "{}",
+            getSyntheticSourceWithFieldLimit(new SourceFilter(null, new String[] { "my_value" }), b -> b.field("my_value", value))
+        );
     }
 
     public void testIgnoredLong() throws IOException {
         long value = randomLong();
         assertEquals("{\"my_value\":" + value + "}", getSyntheticSourceWithFieldLimit(b -> b.field("my_value", value)));
+        assertEquals(
+            "{\"my_value\":" + value + "}",
+            getSyntheticSourceWithFieldLimit(new SourceFilter(new String[] { "my_value" }, null), b -> b.field("my_value", value))
+        );
+        assertEquals(
+            "{}",
+            getSyntheticSourceWithFieldLimit(new SourceFilter(null, new String[] { "my_value" }), b -> b.field("my_value", value))
+        );
     }
 
     public void testIgnoredFloat() throws IOException {
         float value = randomFloat();
         assertEquals("{\"my_value\":" + value + "}", getSyntheticSourceWithFieldLimit(b -> b.field("my_value", value)));
+        assertEquals(
+            "{\"my_value\":" + value + "}",
+            getSyntheticSourceWithFieldLimit(new SourceFilter(new String[] { "my_value" }, null), b -> b.field("my_value", value))
+        );
+        assertEquals(
+            "{}",
+            getSyntheticSourceWithFieldLimit(new SourceFilter(null, new String[] { "my_value" }), b -> b.field("my_value", value))
+        );
     }
 
     public void testIgnoredDouble() throws IOException {
         double value = randomDouble();
         assertEquals("{\"my_value\":" + value + "}", getSyntheticSourceWithFieldLimit(b -> b.field("my_value", value)));
+        assertEquals(
+            "{\"my_value\":" + value + "}",
+            getSyntheticSourceWithFieldLimit(new SourceFilter(new String[] { "my_value" }, null), b -> b.field("my_value", value))
+        );
+        assertEquals(
+            "{}",
+            getSyntheticSourceWithFieldLimit(new SourceFilter(null, new String[] { "my_value" }), b -> b.field("my_value", value))
+        );
     }
 
     public void testIgnoredBigInteger() throws IOException {
         BigInteger value = randomBigInteger();
         assertEquals("{\"my_value\":" + value + "}", getSyntheticSourceWithFieldLimit(b -> b.field("my_value", value)));
+        assertEquals(
+            "{\"my_value\":" + value + "}",
+            getSyntheticSourceWithFieldLimit(new SourceFilter(new String[] { "my_value" }, null), b -> b.field("my_value", value))
+        );
+        assertEquals(
+            "{}",
+            getSyntheticSourceWithFieldLimit(new SourceFilter(null, new String[] { "my_value" }), b -> b.field("my_value", value))
+        );
     }
 
     public void testIgnoredBytes() throws IOException {
@@ -100,6 +193,14 @@ public class IgnoredSourceFieldMapperTests extends MapperServiceTestCase {
             "{\"my_value\":\"" + Base64.getEncoder().encodeToString(value) + "\"}",
             getSyntheticSourceWithFieldLimit(b -> b.field("my_value", value))
         );
+        assertEquals(
+            "{\"my_value\":\"" + Base64.getEncoder().encodeToString(value) + "\"}",
+            getSyntheticSourceWithFieldLimit(new SourceFilter(new String[] { "my_value" }, null), b -> b.field("my_value", value))
+        );
+        assertEquals(
+            "{}",
+            getSyntheticSourceWithFieldLimit(new SourceFilter(null, new String[] { "my_value" }), b -> b.field("my_value", value))
+        );
     }
 
     public void testIgnoredObjectBoolean() throws IOException {
@@ -107,15 +208,124 @@ public class IgnoredSourceFieldMapperTests extends MapperServiceTestCase {
         assertEquals("{\"my_object\":{\"my_value\":" + value + "}}", getSyntheticSourceWithFieldLimit(b -> {
             b.startObject("my_object").field("my_value", value).endObject();
         }));
+
+        assertEquals(
+            "{\"my_object\":{\"my_value\":" + value + "}}",
+            getSyntheticSourceWithFieldLimit(new SourceFilter(new String[] { "my_object" }, null), b -> {
+                b.startObject("my_object").field("my_value", value).endObject();
+            })
+        );
+
+        assertEquals(
+            "{\"my_object\":{\"my_value\":" + value + "}}",
+            getSyntheticSourceWithFieldLimit(new SourceFilter(new String[] { "my_object.my_value" }, null), b -> {
+                b.startObject("my_object").field("my_value", value).endObject();
+            })
+        );
+
+        assertEquals("{}", getSyntheticSourceWithFieldLimit(new SourceFilter(null, new String[] { "my_object" }), b -> {
+            b.startObject("my_object").field("my_value", value).endObject();
+        }));
+
+        assertEquals("{}", getSyntheticSourceWithFieldLimit(new SourceFilter(null, new String[] { "my_object.my_value" }), b -> {
+            b.startObject("my_object").field("my_value", value).endObject();
+        }));
+
+        assertEquals(
+            "{\"my_object\":{\"another_value\":\"0\"}}",
+            getSyntheticSourceWithFieldLimit(new SourceFilter(null, new String[] { "my_object.my_value" }), b -> {
+                b.startObject("my_object").field("my_value", value).field("another_value", "0").endObject();
+            })
+        );
+    }
+
+    public void testIgnoredArrayOfObjects() throws IOException {
+        boolean value = randomBoolean();
+        int another_value = randomInt();
+
+        assertEquals(
+            "{\"my_object\":[{\"my_value\":" + value + "},{\"another_value\":" + another_value + "}]}",
+            getSyntheticSourceWithFieldLimit(b -> {
+                b.startArray("my_object");
+                b.startObject().field("my_value", value).endObject();
+                b.startObject().field("another_value", another_value).endObject();
+                b.endArray();
+            })
+        );
+
+        assertEquals(
+            "{\"my_object\":[{\"another_value\":" + another_value + "}]}",
+            getSyntheticSourceWithFieldLimit(new SourceFilter(null, new String[] { "my_object.my_value" }), b -> {
+                b.startArray("my_object");
+                b.startObject().field("my_value", value).endObject();
+                b.startObject().field("another_value", another_value).endObject();
+                b.endArray();
+            })
+        );
+
+        assertEquals(
+            "{\"my_object\":[{\"my_value\":" + value + "}]}",
+            getSyntheticSourceWithFieldLimit(new SourceFilter(null, new String[] { "my_object.another_value" }), b -> {
+                b.startArray("my_object");
+                b.startObject().field("my_value", value).endObject();
+                b.startObject().field("another_value", another_value).endObject();
+                b.endArray();
+            })
+        );
+
+        assertEquals(
+            "{}",
+            getSyntheticSourceWithFieldLimit(
+                new SourceFilter(null, new String[] { "my_object.another_value", "my_object.my_value" }),
+                b -> {
+                    b.startArray("my_object");
+                    b.startObject().field("my_value", value).endObject();
+                    b.startObject().field("another_value", another_value).endObject();
+                    b.endArray();
+                }
+            )
+        );
+
+        assertEquals(
+            "{\"my_object\":[{\"another_field2\":2}]}",
+            getSyntheticSourceWithFieldLimit(
+                new SourceFilter(null, new String[] { "my_object.another_field1", "my_object.my_value" }),
+                b -> {
+                    b.startArray("my_object");
+                    b.startObject().field("my_value", value).endObject();
+                    b.startObject().field("another_field1", 1).endObject();
+                    b.startObject().field("another_field2", 2).endObject();
+                    b.endArray();
+                }
+            )
+        );
+
+        assertEquals("{}", getSyntheticSourceWithFieldLimit(new SourceFilter(null, new String[] { "my_object" }), b -> {
+            b.startArray("my_object");
+            b.startObject().field("my_value", value).endObject();
+            b.startObject().field("another_value", another_value).endObject();
+            b.endArray();
+        }));
     }
 
     public void testIgnoredArray() throws IOException {
-        assertEquals("{\"my_array\":[{\"int_value\":10},{\"int_value\":20}]}", getSyntheticSourceWithFieldLimit(b -> {
+        assertEquals(
+            "{\"my_array\":[{\"int_value\":10},{\"int_value\":20}]}",
+            getSyntheticSourceWithFieldLimit(new SourceFilter(new String[] { "my_array" }, null), b -> {
+                b.startArray("my_array");
+                b.startObject().field("int_value", 10).endObject();
+                b.startObject().field("int_value", 20).endObject();
+                b.endArray();
+            })
+        );
+
+        assertEquals("{}", getSyntheticSourceWithFieldLimit(new SourceFilter(null, new String[] { "my_array" }), b -> {
             b.startArray("my_array");
             b.startObject().field("int_value", 10).endObject();
             b.startObject().field("int_value", 20).endObject();
             b.endArray();
         }));
+
     }
 
     public void testEncodeFieldToMap() throws IOException {

server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java

@@ -654,8 +654,8 @@ public class ObjectMapperTests extends MapperServiceTestCase {
     public void testSyntheticSourceDocValuesEmpty() throws IOException {
         DocumentMapper mapper = createDocumentMapper(mapping(b -> b.startObject("o").field("type", "object").endObject()));
         ObjectMapper o = (ObjectMapper) mapper.mapping().getRoot().getMapper("o");
-        assertThat(o.syntheticFieldLoader().docValuesLoader(null, null), nullValue());
-        assertThat(mapper.mapping().getRoot().syntheticFieldLoader().docValuesLoader(null, null), nullValue());
+        assertThat(o.syntheticFieldLoader(null).docValuesLoader(null, null), nullValue());
+        assertThat(mapper.mapping().getRoot().syntheticFieldLoader(null).docValuesLoader(null, null), nullValue());
     }
 
     /**
@@ -680,8 +680,8 @@ public class ObjectMapperTests extends MapperServiceTestCase {
             b.endObject().endObject();
         }));
         ObjectMapper o = (ObjectMapper) mapper.mapping().getRoot().getMapper("o");
-        assertThat(o.syntheticFieldLoader().docValuesLoader(null, null), nullValue());
-        assertThat(mapper.mapping().getRoot().syntheticFieldLoader().docValuesLoader(null, null), nullValue());
+        assertThat(o.syntheticFieldLoader(null).docValuesLoader(null, null), nullValue());
+        assertThat(mapper.mapping().getRoot().syntheticFieldLoader(null).docValuesLoader(null, null), nullValue());
     }
 
     public void testStoreArraySourceinSyntheticSourceMode() throws IOException {

server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java

@@ -408,7 +408,7 @@ public abstract class RangeFieldMapperTests extends MapperTestCase {
             iw.addDocument(doc);
             iw.close();
             try (DirectoryReader reader = DirectoryReader.open(directory)) {
-                SourceProvider provider = SourceProvider.fromSyntheticSource(mapper.mapping(), SourceFieldMetrics.NOOP);
+                SourceProvider provider = SourceProvider.fromSyntheticSource(mapper.mapping(), null, SourceFieldMetrics.NOOP);
                 Source syntheticSource = provider.getSource(getOnlyLeafReader(reader).getContext(), 0);
 
                 return syntheticSource;

server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMetricsTests.java

@@ -47,6 +47,7 @@ public class SourceFieldMetricsTests extends MapperServiceTestCase {
             try (DirectoryReader reader = DirectoryReader.open(directory)) {
                 SourceProvider provider = SourceProvider.fromSyntheticSource(
                     mapper.mapping(),
+                    null,
                     createTestMapperMetrics().sourceFieldMetrics()
                 );
                 Source synthetic = provider.getSource(getOnlyLeafReader(reader).getContext(), 0);

server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java

@@ -21,7 +21,7 @@ public class SourceLoaderTests extends MapperServiceTestCase {
             b.startObject("o").field("type", "object").endObject();
             b.startObject("kwd").field("type", "keyword").endObject();
         }));
-        assertFalse(mapper.mappers().newSourceLoader(SourceFieldMetrics.NOOP).reordersFieldValues());
+        assertFalse(mapper.mappers().newSourceLoader(null, SourceFieldMetrics.NOOP).reordersFieldValues());
     }
 
     public void testEmptyObject() throws IOException {
@@ -29,7 +29,7 @@ public class SourceLoaderTests extends MapperServiceTestCase {
             b.startObject("o").field("type", "object").endObject();
             b.startObject("kwd").field("type", "keyword").endObject();
         })).documentMapper();
-        assertTrue(mapper.mappers().newSourceLoader(SourceFieldMetrics.NOOP).reordersFieldValues());
+        assertTrue(mapper.mappers().newSourceLoader(null, SourceFieldMetrics.NOOP).reordersFieldValues());
         assertThat(syntheticSource(mapper, b -> b.field("kwd", "foo")), equalTo("""
             {"kwd":"foo"}"""));
     }

server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java

@@ -1354,7 +1354,7 @@ public class TextFieldMapperTests extends MapperTestCase {
         XContentBuilder mapping = mapping(buildFields);
         MapperService mapper = syntheticSource ? createSytheticSourceMapperService(mapping) : createMapperService(mapping);
         BlockReaderSupport blockReaderSupport = getSupportedReaders(mapper, "field.sub");
-        var sourceLoader = mapper.mappingLookup().newSourceLoader(SourceFieldMetrics.NOOP);
+        var sourceLoader = mapper.mappingLookup().newSourceLoader(null, SourceFieldMetrics.NOOP);
         testBlockLoader(columnReader, example, blockReaderSupport, sourceLoader);
     }
 }

server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java

@@ -18,6 +18,7 @@ import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xcontent.json.JsonXContent;
 
@@ -29,6 +30,7 @@ import java.util.Base64;
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.stream.Stream;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -57,6 +59,7 @@ public class XContentDataHelperTests extends ESTestCase {
         parser.nextToken();
 
         var encoded = XContentDataHelper.encodeToken(parser);
+        assertThat(XContentDataHelper.isEncodedObject(encoded), equalTo(value instanceof Map));
         var decoded = XContentFactory.jsonBuilder();
         XContentDataHelper.decodeAndWrite(decoded, encoded);
 
@@ -124,6 +127,7 @@ public class XContentDataHelperTests extends ESTestCase {
             assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
             parser.nextToken();
             var encoded = XContentDataHelper.encodeToken(parser);
+            assertFalse(XContentDataHelper.isEncodedObject(encoded));
 
             var decoded = XContentFactory.jsonBuilder();
             XContentDataHelper.decodeAndWrite(decoded, encoded);
@@ -132,6 +136,7 @@ public class XContentDataHelperTests extends ESTestCase {
         }
 
         var encoded = XContentDataHelper.encodeXContentBuilder(builder);
+        assertTrue(XContentDataHelper.isEncodedObject(encoded));
 
         var decoded = XContentFactory.jsonBuilder();
         XContentDataHelper.decodeAndWrite(decoded, encoded);
@@ -147,7 +152,9 @@ public class XContentDataHelperTests extends ESTestCase {
 
         XContentBuilder builder = XContentFactory.jsonBuilder();
         builder.humanReadable(true);
-        XContentDataHelper.decodeAndWrite(builder, XContentDataHelper.encodeToken(p));
+        var encoded = XContentDataHelper.encodeToken(p);
+        assertTrue(XContentDataHelper.isEncodedObject(encoded));
+        XContentDataHelper.decodeAndWrite(builder, encoded);
         assertEquals(object, Strings.toString(builder));
 
         XContentBuilder builder2 = XContentFactory.jsonBuilder();
@@ -156,6 +163,62 @@ public class XContentDataHelperTests extends ESTestCase {
         assertEquals(object, Strings.toString(builder2));
     }
 
+    public void testObjectWithFilter() throws IOException {
+        String object = "{\"name\":\"foo\",\"path\":{\"filter\":{\"keep\":[0],\"field\":\"value\"}}}";
+        String filterObject = "{\"name\":\"foo\",\"path\":{\"filter\":{\"keep\":[0]}}}";
+
+        XContentParser p = createParser(JsonXContent.jsonXContent, object);
+        assertThat(p.nextToken(), equalTo(XContentParser.Token.START_OBJECT));
+        XContentParserConfiguration parserConfig = XContentParserConfiguration.EMPTY.withFiltering(
+            null,
+            null,
+            Set.of("path.filter.field"),
+            true
+        );
+        XContentBuilder builder = XContentFactory.jsonBuilder();
+        builder.humanReadable(true);
+        XContentDataHelper.decodeAndWriteXContent(parserConfig, builder, XContentType.JSON, XContentDataHelper.encodeToken(p));
+        assertEquals(filterObject, Strings.toString(builder));
+
+        XContentBuilder builder2 = XContentFactory.jsonBuilder();
+        builder2.humanReadable(true);
+        XContentDataHelper.decodeAndWriteXContent(
+            parserConfig,
+            builder2,
+            XContentType.JSON,
+            XContentDataHelper.encodeXContentBuilder(builder)
+        );
+        assertEquals(filterObject, Strings.toString(builder2));
+    }
+
+    public void testObjectWithFilterRootPath() throws IOException {
+        String object = "{\"name\":\"foo\",\"path\":{\"filter\":{\"keep\":[0],\"field\":\"value\"}}}";
+        String filterObject = "{\"path\":{\"filter\":{\"keep\":[0]}}}";
+
+        XContentParser p = createParser(JsonXContent.jsonXContent, object);
+        assertThat(p.nextToken(), equalTo(XContentParser.Token.START_OBJECT));
+        XContentParserConfiguration parserConfig = XContentParserConfiguration.EMPTY.withFiltering(
+            "root.obj.sub_obj",
+            Set.of("root.obj.sub_obj.path"),
+            Set.of("root.obj.sub_obj.path.filter.field"),
+            true
+        );
+        XContentBuilder builder = XContentFactory.jsonBuilder();
+        builder.humanReadable(true);
+        XContentDataHelper.decodeAndWriteXContent(parserConfig, builder, XContentType.JSON, XContentDataHelper.encodeToken(p));
+        assertEquals(filterObject, Strings.toString(builder));
+
+        XContentBuilder builder2 = XContentFactory.jsonBuilder();
+        builder2.humanReadable(true);
+        XContentDataHelper.decodeAndWriteXContent(
+            parserConfig,
+            builder2,
+            XContentType.JSON,
+            XContentDataHelper.encodeXContentBuilder(builder)
+        );
+        assertEquals(filterObject, Strings.toString(builder2));
+    }
+
     public void testArrayInt() throws IOException {
         String values = "["
             + String.join(",", List.of(Integer.toString(randomInt()), Integer.toString(randomInt()), Integer.toString(randomInt())))
@@ -252,7 +315,7 @@ public class XContentDataHelperTests extends ESTestCase {
 
         var destination = XContentFactory.contentBuilder(xContentType);
         destination.startObject();
-        XContentDataHelper.writeMerged(destination, "foo", List.of(firstEncoded, secondEncoded));
+        XContentDataHelper.writeMerged(XContentParserConfiguration.EMPTY, destination, "foo", List.of(firstEncoded, secondEncoded));
         destination.endObject();
 
         return XContentHelper.convertToMap(BytesReference.bytes(destination), false, xContentType).v2();

test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java

@@ -70,6 +70,7 @@ import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry;
 import org.elasticsearch.search.internal.SubSearchContext;
 import org.elasticsearch.search.lookup.SearchLookup;
+import org.elasticsearch.search.lookup.SourceFilter;
 import org.elasticsearch.search.lookup.SourceProvider;
 import org.elasticsearch.search.sort.BucketedSort;
 import org.elasticsearch.search.sort.BucketedSort.ExtraData;
@@ -798,6 +799,14 @@ public abstract class MapperServiceTestCase extends FieldTypeTestCase {
     }
 
     protected final String syntheticSource(DocumentMapper mapper, CheckedConsumer<XContentBuilder, IOException> build) throws IOException {
+        return syntheticSource(mapper, null, build);
+    }
+
+    protected final String syntheticSource(
+        DocumentMapper mapper,
+        @Nullable SourceFilter sourceFilter,
+        CheckedConsumer<XContentBuilder, IOException> build
+    ) throws IOException {
         try (Directory directory = newDirectory()) {
             RandomIndexWriter iw = indexWriterForSyntheticSource(directory);
             ParsedDocument doc = mapper.parse(source(build));
@@ -806,9 +815,10 @@ public abstract class MapperServiceTestCase extends FieldTypeTestCase {
             iw.addDocuments(doc.docs());
             iw.close();
             try (DirectoryReader indexReader = wrapInMockESDirectoryReader(DirectoryReader.open(directory))) {
-                String syntheticSource = syntheticSource(mapper, indexReader, doc.docs().size() - 1);
+                String syntheticSourceFiltered = syntheticSource(mapper, sourceFilter, indexReader, doc.docs().size() - 1);
+                String syntheticSource = syntheticSource(mapper, null, indexReader, doc.docs().size() - 1);
                 roundTripSyntheticSource(mapper, syntheticSource, indexReader);
-                return syntheticSource;
+                return syntheticSourceFiltered;
             }
         }
     }
@@ -841,12 +851,16 @@ public abstract class MapperServiceTestCase extends FieldTypeTestCase {
     }
 
     protected static String syntheticSource(DocumentMapper mapper, IndexReader reader, int docId) throws IOException {
+        return syntheticSource(mapper, null, reader, docId);
+    }
+
+    protected static String syntheticSource(DocumentMapper mapper, SourceFilter filter, IndexReader reader, int docId) throws IOException {
         LeafReader leafReader = getOnlyLeafReader(reader);
 
         final String synthetic1;
         final XContent xContent;
         {
-            SourceProvider provider = SourceProvider.fromSyntheticSource(mapper.mapping(), SourceFieldMetrics.NOOP);
+            SourceProvider provider = SourceProvider.fromSyntheticSource(mapper.mapping(), filter, SourceFieldMetrics.NOOP);
             var source = provider.getSource(leafReader.getContext(), docId);
             synthetic1 = source.internalSourceRef().utf8ToString();
             xContent = source.sourceContentType().xContent();
@@ -855,7 +869,11 @@ public abstract class MapperServiceTestCase extends FieldTypeTestCase {
         final String synthetic2;
         {
             int[] docIds = new int[] { docId };
-            SourceLoader sourceLoader = new SourceLoader.Synthetic(mapper.mapping()::syntheticFieldLoader, SourceFieldMetrics.NOOP);
+            SourceLoader sourceLoader = new SourceLoader.Synthetic(
+                filter,
+                () -> mapper.mapping().syntheticFieldLoader(filter),
+                SourceFieldMetrics.NOOP
+            );
             var sourceLeafLoader = sourceLoader.leaf(getOnlyLeafReader(reader), docIds);
             var storedFieldLoader = StoredFieldLoader.create(false, sourceLoader.requiredStoredFields())
                 .getLoader(leafReader.getContext(), docIds);

test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java

@@ -64,6 +64,7 @@ import org.elasticsearch.search.fetch.StoredFieldsSpec;
 import org.elasticsearch.search.lookup.LeafStoredFieldsLookup;
 import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.search.lookup.Source;
+import org.elasticsearch.search.lookup.SourceFilter;
 import org.elasticsearch.search.lookup.SourceProvider;
 import org.elasticsearch.test.ListMatcher;
 import org.elasticsearch.xcontent.ToXContent;
@@ -1185,6 +1186,11 @@ public abstract class MapperTestCase extends MapperServiceTestCase {
             b.endObject();
         })).documentMapper();
         assertThat(syntheticSource(mapper, example::buildInput), equalTo(example.expected()));
+        assertThat(
+            syntheticSource(mapper, new SourceFilter(new String[] { "field" }, null), example::buildInput),
+            equalTo(example.expected())
+        );
+        assertThat(syntheticSource(mapper, new SourceFilter(null, new String[] { "field" }), example::buildInput), equalTo("{}"));
     }
 
     private void assertSyntheticSourceWithTranslogSnapshot(SyntheticSourceSupport support, boolean doIndexSort) throws IOException {
@@ -1308,7 +1314,7 @@ public abstract class MapperTestCase extends MapperServiceTestCase {
             }
             try (DirectoryReader reader = DirectoryReader.open(directory)) {
                 int i = 0;
-                SourceLoader loader = mapper.sourceMapper().newSourceLoader(mapper.mapping(), SourceFieldMetrics.NOOP);
+                SourceLoader loader = mapper.mappers().newSourceLoader(null, SourceFieldMetrics.NOOP);
                 StoredFieldLoader storedFieldLoader = loader.requiredStoredFields().isEmpty()
                     ? StoredFieldLoader.empty()
                     : StoredFieldLoader.create(false, loader.requiredStoredFields());
@@ -1351,6 +1357,18 @@ public abstract class MapperTestCase extends MapperServiceTestCase {
             syntheticSourceExample.buildInput(b);
             b.endObject();
         }), equalTo("{\"obj\":" + syntheticSourceExample.expected() + "}"));
+
+        assertThat(syntheticSource(mapper, new SourceFilter(new String[] { "obj.field" }, null), b -> {
+            b.startObject("obj");
+            syntheticSourceExample.buildInput(b);
+            b.endObject();
+        }), equalTo("{\"obj\":" + syntheticSourceExample.expected() + "}"));
+
+        assertThat(syntheticSource(mapper, new SourceFilter(null, new String[] { "obj.field" }), b -> {
+            b.startObject("obj");
+            syntheticSourceExample.buildInput(b);
+            b.endObject();
+        }), equalTo("{}"));
     }
 
     public final void testSyntheticEmptyList() throws IOException {
@@ -1481,7 +1499,7 @@ public abstract class MapperTestCase extends MapperServiceTestCase {
                 blockReaderSupport.syntheticSource
             );
         }
-        var sourceLoader = mapper.mappingLookup().newSourceLoader(SourceFieldMetrics.NOOP);
+        var sourceLoader = mapper.mappingLookup().newSourceLoader(null, SourceFieldMetrics.NOOP);
         testBlockLoader(columnReader, example, blockReaderSupport, sourceLoader);
     }
 
@@ -1608,7 +1626,8 @@ public abstract class MapperTestCase extends MapperServiceTestCase {
             iw.close();
             try (DirectoryReader reader = DirectoryReader.open(directory)) {
                 LeafReader leafReader = getOnlyLeafReader(reader);
-                SourceLoader.SyntheticFieldLoader fieldLoader = mapper.mapping().getRoot().getMapper("field").syntheticFieldLoader();
+                SourceLoader.SyntheticFieldLoader fieldLoader = ((FieldMapper) mapper.mapping().getRoot().getMapper("field"))
+                    .syntheticFieldLoader();
                 /*
                  * null means "there are no values for this field, don't call me".
                  * Empty fields are common enough that we need to make sure this
@@ -1649,6 +1668,24 @@ public abstract class MapperTestCase extends MapperServiceTestCase {
             syntheticSourceExample.buildInput(b);
             b.endObject();
         }), equalTo("{\"obj\":" + syntheticSourceExample.expected() + "}"));
+
+        assertThat(syntheticSource(mapper, new SourceFilter(new String[] { "obj.field" }, null), b -> {
+            b.startObject("obj");
+            syntheticSourceExample.buildInput(b);
+            b.endObject();
+        }), equalTo("{\"obj\":" + syntheticSourceExample.expected() + "}"));
+
+        assertThat(syntheticSource(mapper, new SourceFilter(null, new String[] { "obj.field" }), b -> {
+            b.startObject("obj");
+            syntheticSourceExample.buildInput(b);
+            b.endObject();
+        }), equalTo("{\"obj\":{}}"));
+
+        assertThat(syntheticSource(mapper, new SourceFilter(null, new String[] { "obj" }), b -> {
+            b.startObject("obj");
+            syntheticSourceExample.buildInput(b);
+            b.endObject();
+        }), equalTo("{}"));
     }
 
     protected SyntheticSourceSupport syntheticSourceSupportForKeepTests(boolean ignoreMalformed) {

x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java

@@ -68,6 +68,7 @@ public class TransportPutRollupJobAction extends AcknowledgedTransportMasterNode
 
     private static final Logger LOGGER = LogManager.getLogger(TransportPutRollupJobAction.class);
     private static final XContentParserConfiguration PARSER_CONFIGURATION = XContentParserConfiguration.EMPTY.withFiltering(
+        null,
         Set.of("_doc._meta._rollup"),
         null,
         false