Add exclusion filters support to filter_path

This commit adds the support for exclusion filter to the response filtering (filter_path) feature. It changes the XContentBuilder APIs so that it now accepts two types of filters: inclusive and exclusive. Filters are no more String arrays but sets of String instead.

core/src/main/java/org/elasticsearch/common/xcontent/XContent.java

@@ -19,12 +19,15 @@
 
 package org.elasticsearch.common.xcontent;
 
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.Reader;
+import java.util.Collections;
+import java.util.Set;
 
 /**
  * A generic abstraction on top of handling content, inspired by JSON and pull parsing.
@@ -42,27 +45,20 @@ public interface XContent {
      * Creates a new generator using the provided output stream.
      */
     default XContentGenerator createGenerator(OutputStream os) throws IOException {
-        return createGenerator(os, null, true);
+        return createGenerator(os, Collections.emptySet(), Collections.emptySet());
     }
 
     /**
-     * Creates a new generator using the provided output stream and some
-     * inclusive filters. Same as createGenerator(os, filters, true).
-     */
-    default XContentGenerator createGenerator(OutputStream os, String[] filters) throws IOException {
-        return createGenerator(os, filters, true);
-    }
-
-    /**
-     * Creates a new generator using the provided output stream and some
-     * filters.
+     * Creates a new generator using the provided output stream and some inclusive and/or exclusive filters. When both exclusive and
+     * inclusive filters are provided, the underlying generator will first use exclusion filters to remove fields and then will check the
+     * remaining fields against the inclusive filters.
      *
-     * @param inclusive
-     *            If true only paths matching a filter will be included in
-     *            output. If false no path matching a filter will be included in
-     *            output
+     * @param os       the output stream
+     * @param includes the inclusive filters: only fields and objects that match the inclusive filters will be written to the output.
+     * @param excludes the exclusive filters: only fields and objects that don't match the exclusive filters will be written to the output.
      */
-    XContentGenerator createGenerator(OutputStream os, String[] filters, boolean inclusive) throws IOException;
+    XContentGenerator createGenerator(OutputStream os, Set<String> includes, Set<String> excludes) throws IOException;
+
     /**
      * Creates a parser over the provided string content.
      */

core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java

@@ -19,21 +19,8 @@
 
 package org.elasticsearch.common.xcontent;
 
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.math.BigDecimal;
-import java.math.RoundingMode;
-import java.nio.file.Path;
-import java.util.Calendar;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Locale;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.io.BytesStream;
@@ -47,6 +34,21 @@ import org.joda.time.ReadableInstant;
 import org.joda.time.format.DateTimeFormatter;
 import org.joda.time.format.ISODateTimeFormat;
 
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.math.BigDecimal;
+import java.math.RoundingMode;
+import java.nio.file.Path;
+import java.util.Calendar;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
 /**
  * A utility to build XContent (ie json).
  */
@@ -58,12 +60,8 @@ public final class XContentBuilder implements BytesStream, Releasable {
         return new XContentBuilder(xContent, new BytesStreamOutput());
     }
 
-    public static XContentBuilder builder(XContent xContent, String[] filters) throws IOException {
-        return new XContentBuilder(xContent, new BytesStreamOutput(), filters);
-    }
-
-    public static XContentBuilder builder(XContent xContent, String[] filters, boolean inclusive) throws IOException {
-        return new XContentBuilder(xContent, new BytesStreamOutput(), filters, inclusive);
+    public static XContentBuilder builder(XContent xContent, Set<String> includes, Set<String> excludes) throws IOException {
+        return new XContentBuilder(xContent, new BytesStreamOutput(), includes, excludes);
     }
 
     private XContentGenerator generator;
@@ -77,7 +75,7 @@ public final class XContentBuilder implements BytesStream, Releasable {
      * to call {@link #close()} when the builder is done with.
      */
     public XContentBuilder(XContent xContent, OutputStream bos) throws IOException {
-        this(xContent, bos, null);
+        this(xContent, bos, Collections.emptySet(), Collections.emptySet());
     }
 
     /**
@@ -86,20 +84,24 @@ public final class XContentBuilder implements BytesStream, Releasable {
      * filter will be written to the output stream. Make sure to call
      * {@link #close()} when the builder is done with.
      */
-    public XContentBuilder(XContent xContent, OutputStream bos, String[] filters) throws IOException {
-        this(xContent, bos, filters, true);
+    public XContentBuilder(XContent xContent, OutputStream bos, Set<String> includes) throws IOException {
+        this(xContent, bos, includes, Collections.emptySet());
     }
 
     /**
-     * Constructs a new builder using the provided xcontent, an OutputStream and
-     * some filters. If {@code filters} are specified and {@code inclusive} is
-     * true, only those values matching a filter will be written to the output
-     * stream. If {@code inclusive} is false, those matching will be excluded.
+     * Creates a new builder using the provided XContent, output stream and some inclusive and/or exclusive filters. When both exclusive and
+     * inclusive filters are provided, the underlying builder will first use exclusion filters to remove fields and then will check the
+     * remaining fields against the inclusive filters.
+     * <p>
      * Make sure to call {@link #close()} when the builder is done with.
+     *
+     * @param os       the output stream
+     * @param includes the inclusive filters: only fields and objects that match the inclusive filters will be written to the output.
+     * @param excludes the exclusive filters: only fields and objects that don't match the exclusive filters will be written to the output.
      */
-    public XContentBuilder(XContent xContent, OutputStream bos, String[] filters, boolean inclusive) throws IOException {
-        this.bos = bos;
-        this.generator = xContent.createGenerator(bos, filters, inclusive);
+    public XContentBuilder(XContent xContent, OutputStream os, Set<String> includes, Set<String> excludes) throws IOException {
+        this.bos = os;
+        this.generator = xContent.createGenerator(bos, includes, excludes);
     }
 
     public XContentType contentType() {

core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java

@@ -35,6 +35,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.Reader;
+import java.util.Set;
 
 /**
  * A CBOR based content implementation using Jackson.
@@ -70,8 +71,8 @@ public class CborXContent implements XContent {
     }
 
     @Override
-    public XContentGenerator createGenerator(OutputStream os, String[] filters, boolean inclusive) throws IOException {
-        return new CborXContentGenerator(cborFactory.createGenerator(os, JsonEncoding.UTF8), os, filters, inclusive);
+    public XContentGenerator createGenerator(OutputStream os, Set<String> includes, Set<String> excludes) throws IOException {
+        return new CborXContentGenerator(cborFactory.createGenerator(os, JsonEncoding.UTF8), os, includes, excludes);
     }
 
     @Override

core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentGenerator.java

@@ -20,23 +20,22 @@
 package org.elasticsearch.common.xcontent.cbor;
 
 import com.fasterxml.jackson.core.JsonGenerator;
-
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContentGenerator;
 
 import java.io.OutputStream;
+import java.util.Collections;
+import java.util.Set;
 
-/**
- *
- */
 public class CborXContentGenerator extends JsonXContentGenerator {
 
-    public CborXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, String... filters) {
-        this(jsonGenerator, os, filters, true);
+    public CborXContentGenerator(JsonGenerator jsonGenerator, OutputStream os) {
+        this(jsonGenerator, os, Collections.emptySet(), Collections.emptySet());
     }
 
-    public CborXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, String[] filters, boolean inclusive) {
-        super(jsonGenerator, os, filters, inclusive);
+    public CborXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, Set<String> includes, Set<String> excludes) {
+        super(jsonGenerator, os, includes, excludes);
     }
 
     @Override

core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java

@@ -35,6 +35,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.Reader;
+import java.util.Set;
 
 /**
  * A JSON based content implementation using Jackson.
@@ -92,8 +93,8 @@ public class JsonXContent implements XContent {
     }
 
     @Override
-    public XContentGenerator createGenerator(OutputStream os, String[] filters, boolean inclusive) throws IOException {
-        return new JsonXContentGenerator(jsonFactory.createGenerator(os, JsonEncoding.UTF8), os, filters, inclusive);
+    public XContentGenerator createGenerator(OutputStream os, Set<String> includes, Set<String> excludes) throws IOException {
+        return new JsonXContentGenerator(jsonFactory.createGenerator(os, JsonEncoding.UTF8), os, includes, excludes);
     }
 
     @Override

core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java

@@ -27,10 +27,10 @@ import com.fasterxml.jackson.core.io.SerializedString;
 import com.fasterxml.jackson.core.json.JsonWriteContext;
 import com.fasterxml.jackson.core.util.DefaultIndenter;
 import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
+import com.fasterxml.jackson.core.util.JsonGeneratorDelegate;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentGenerator;
@@ -43,6 +43,9 @@ import java.io.BufferedInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.util.Collections;
+import java.util.Objects;
+import java.util.Set;
 
 /**
  *
@@ -72,23 +75,38 @@ public class JsonXContentGenerator implements XContentGenerator {
     private static final DefaultPrettyPrinter.Indenter INDENTER = new DefaultIndenter("  ", LF.getValue());
     private boolean prettyPrint = false;
 
-    public JsonXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, String[] filters, boolean inclusive) {
+    public JsonXContentGenerator(JsonGenerator jsonGenerator, OutputStream os) {
+        this(jsonGenerator, os, Collections.emptySet(), Collections.emptySet());
+    }
+
+    public JsonXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, Set<String> includes, Set<String> excludes) {
+        Objects.requireNonNull(includes, "Including filters must not be null");
+        Objects.requireNonNull(excludes, "Excluding filters must not be null");
+        this.os = os;
         if (jsonGenerator instanceof GeneratorBase) {
             this.base = (GeneratorBase) jsonGenerator;
         } else {
             this.base = null;
         }
 
-        if (CollectionUtils.isEmpty(filters)) {
-            this.generator = jsonGenerator;
-            this.filter = null;
-        } else {
-            this.filter = new FilteringGeneratorDelegate(jsonGenerator,
-                    new FilterPathBasedFilter(filters, inclusive), true, true);
-            this.generator = this.filter;
+        JsonGenerator generator = jsonGenerator;
+
+        boolean hasExcludes = excludes.isEmpty() == false;
+        if (hasExcludes) {
+            generator = new FilteringGeneratorDelegate(generator, new FilterPathBasedFilter(excludes, false), true, true);
         }
 
-        this.os = os;
+        boolean hasIncludes = includes.isEmpty() == false;
+        if (hasIncludes) {
+            generator = new FilteringGeneratorDelegate(generator, new FilterPathBasedFilter(includes, true), true, true);
+        }
+
+        if (hasExcludes || hasIncludes) {
+            this.filter = (FilteringGeneratorDelegate) generator;
+        } else {
+            this.filter = null;
+        }
+        this.generator = generator;
     }
 
     @Override
@@ -122,23 +140,34 @@ public class JsonXContentGenerator implements XContentGenerator {
         generator.writeEndArray();
     }
 
-    protected boolean isFiltered() {
+    private boolean isFiltered() {
         return filter != null;
     }
 
-    protected boolean inRoot() {
+    private JsonGenerator getLowLevelGenerator() {
         if (isFiltered()) {
-            JsonStreamContext context = filter.getFilterContext();
-            return ((context != null) && (context.inRoot() && context.getCurrentName() == null));
+            JsonGenerator delegate = filter.getDelegate();
+            if (delegate instanceof JsonGeneratorDelegate) {
+                // In case of combined inclusion and exclusion filters, we have one and only one another delegating level
+                delegate = ((JsonGeneratorDelegate) delegate).getDelegate();
+                assert delegate instanceof JsonGeneratorDelegate == false;
+            }
+            return delegate;
         }
-        return false;
+        return generator;
+    }
+
+    private boolean inRoot() {
+        JsonStreamContext context = generator.getOutputContext();
+        return ((context != null) && (context.inRoot() && context.getCurrentName() == null));
     }
 
     @Override
     public void writeStartObject() throws IOException {
-        if (isFiltered() && inRoot()) {
-            // Bypass generator to always write the root start object
-            filter.getDelegate().writeStartObject();
+        if (inRoot()) {
+            // Use the low level generator to write the startObject so that the root
+            // start object is always written even if a filtered generator is used
+            getLowLevelGenerator().writeStartObject();
             return;
         }
         generator.writeStartObject();
@@ -146,9 +175,10 @@ public class JsonXContentGenerator implements XContentGenerator {
 
     @Override
     public void writeEndObject() throws IOException {
-        if (isFiltered() && inRoot()) {
-            // Bypass generator to always write the root end object
-            filter.getDelegate().writeEndObject();
+        if (inRoot()) {
+            // Use the low level generator to write the startObject so that the root
+            // start object is always written even if a filtered generator is used
+            getLowLevelGenerator().writeEndObject();
             return;
         }
         generator.writeEndObject();
@@ -390,7 +420,8 @@ public class JsonXContentGenerator implements XContentGenerator {
         }
         if (writeLineFeedAtEnd) {
             flush();
-            generator.writeRaw(LF);
+            // Bypass generator to always write the line feed
+            getLowLevelGenerator().writeRaw(LF);
         }
         generator.close();
     }

core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java

@@ -35,6 +35,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.Reader;
+import java.util.Set;
 
 /**
  * A Smile based content implementation using Jackson.
@@ -71,8 +72,8 @@ public class SmileXContent implements XContent {
     }
 
     @Override
-    public XContentGenerator createGenerator(OutputStream os, String[] filters, boolean inclusive) throws IOException {
-        return new SmileXContentGenerator(smileFactory.createGenerator(os, JsonEncoding.UTF8), os, filters, inclusive);
+    public XContentGenerator createGenerator(OutputStream os, Set<String> includes, Set<String> excludes) throws IOException {
+        return new SmileXContentGenerator(smileFactory.createGenerator(os, JsonEncoding.UTF8), os, includes, excludes);
     }
 
     @Override

core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentGenerator.java

@@ -20,23 +20,22 @@
 package org.elasticsearch.common.xcontent.smile;
 
 import com.fasterxml.jackson.core.JsonGenerator;
-
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContentGenerator;
 
 import java.io.OutputStream;
+import java.util.Collections;
+import java.util.Set;
 
-/**
- *
- */
 public class SmileXContentGenerator extends JsonXContentGenerator {
 
-    public SmileXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, String... filters) {
-        this(jsonGenerator, os, filters, true);
+    public SmileXContentGenerator(JsonGenerator jsonGenerator, OutputStream os) {
+        this(jsonGenerator, os, Collections.emptySet(), Collections.emptySet());
     }
 
-    public SmileXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, String[] filters, boolean inclusive) {
-        super(jsonGenerator, os, filters, inclusive);
+    public SmileXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, Set<String> includes, Set<String> excludes) {
+        super(jsonGenerator, os, includes, excludes);
     }
 
     @Override

core/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilterPath.java

@@ -21,10 +21,10 @@
 package org.elasticsearch.common.xcontent.support.filtering;
 
 import org.elasticsearch.common.regex.Regex;
-import org.elasticsearch.common.util.CollectionUtils;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
 
 public class FilterPath {
 
@@ -75,8 +75,8 @@ public class FilterPath {
         return next;
     }
 
-    public static FilterPath[] compile(String... filters) {
-        if (CollectionUtils.isEmpty(filters)) {
+    public static FilterPath[] compile(Set<String> filters) {
+        if (filters == null || filters.isEmpty()) {
             return null;
         }
 

core/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathBasedFilter.java

@@ -24,6 +24,7 @@ import org.elasticsearch.common.util.CollectionUtils;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
 
 public class FilterPathBasedFilter extends TokenFilter {
 
@@ -53,7 +54,7 @@ public class FilterPathBasedFilter extends TokenFilter {
         this.filters = filters;
     }
 
-    public FilterPathBasedFilter(String[] filters, boolean inclusive) {
+    public FilterPathBasedFilter(Set<String> filters, boolean inclusive) {
         this(FilterPath.compile(filters), inclusive);
     }
 
@@ -103,11 +104,6 @@ public class FilterPathBasedFilter extends TokenFilter {
 
     @Override
     protected boolean _includeScalar() {
-        for (FilterPath filter : filters) {
-            if (filter.matches()) {
-                return inclusive;
-            }
-        }
         return !inclusive;
     }
 }

core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java

@@ -34,6 +34,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.Reader;
+import java.util.Set;
 
 /**
  * A YAML based content implementation using Jackson.
@@ -66,8 +67,8 @@ public class YamlXContent implements XContent {
     }
 
     @Override
-    public XContentGenerator createGenerator(OutputStream os, String[] filters, boolean inclusive) throws IOException {
-        return new YamlXContentGenerator(yamlFactory.createGenerator(os, JsonEncoding.UTF8), os, filters, inclusive);
+    public XContentGenerator createGenerator(OutputStream os, Set<String> includes, Set<String> excludes) throws IOException {
+        return new YamlXContentGenerator(yamlFactory.createGenerator(os, JsonEncoding.UTF8), os, includes, excludes);
     }
 
     @Override

core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentGenerator.java

@@ -20,23 +20,22 @@
 package org.elasticsearch.common.xcontent.yaml;
 
 import com.fasterxml.jackson.core.JsonGenerator;
-
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContentGenerator;
 
 import java.io.OutputStream;
+import java.util.Collections;
+import java.util.Set;
 
-/**
- *
- */
 public class YamlXContentGenerator extends JsonXContentGenerator {
 
-    public YamlXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, String... filters) {
-        this(jsonGenerator, os, filters, true);
+    public YamlXContentGenerator(JsonGenerator jsonGenerator, OutputStream os) {
+        this(jsonGenerator, os, Collections.emptySet(), Collections.emptySet());
     }
 
-    public YamlXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, String[] filters, boolean inclusive) {
-        super(jsonGenerator, os, filters, inclusive);
+    public YamlXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, Set<String> includes, Set<String> excludes) {
+        super(jsonGenerator, os, includes, excludes);
     }
 
     @Override

core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java

@@ -19,6 +19,7 @@
 package org.elasticsearch.rest;
 
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -26,9 +27,17 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
+import java.util.Collections;
+import java.util.Set;
+import java.util.function.Predicate;
+
+import static java.util.stream.Collectors.toSet;
 
 public abstract class AbstractRestChannel implements RestChannel {
 
+    private static final Predicate<String> INCLUDE_FILTER = f -> f.charAt(0) != '-';
+    private static final Predicate<String> EXCLUDE_FILTER = INCLUDE_FILTER.negate();
+
     protected final RestRequest request;
     protected final boolean detailedErrorsEnabled;
 
@@ -41,7 +50,7 @@ public abstract class AbstractRestChannel implements RestChannel {
 
     @Override
     public XContentBuilder newBuilder() throws IOException {
-        return newBuilder(request.hasContent() ? request.content() : null, request.hasParam("filter_path"));
+        return newBuilder(request.hasContent() ? request.content() : null, true);
     }
 
     @Override
@@ -64,8 +73,15 @@ public abstract class AbstractRestChannel implements RestChannel {
             contentType = XContentType.JSON;
         }
 
-        String[] filters = useFiltering ? request.paramAsStringArrayOrEmptyIfAll("filter_path") :  null;
-        XContentBuilder builder = new XContentBuilder(XContentFactory.xContent(contentType), bytesOutput(), filters);
+        Set<String> includes = Collections.emptySet();
+        Set<String> excludes = Collections.emptySet();
+        if (useFiltering) {
+            Set<String> filters = Strings.splitStringByCommaToSet(request.param("filter_path", null));
+            includes = filters.stream().filter(INCLUDE_FILTER).collect(toSet());
+            excludes = filters.stream().filter(EXCLUDE_FILTER).map(f -> f.substring(1)).collect(toSet());
+        }
+
+        XContentBuilder builder = new XContentBuilder(XContentFactory.xContent(contentType), bytesOutput(), includes, excludes);
         if (request.paramAsBoolean("pretty", false)) {
             builder.prettyPrint().lfAtEnd();
         }

core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractFilteringJsonGeneratorTestCase.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.common.xcontent.support.filtering;
 
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -28,7 +29,11 @@ import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
+import java.util.Set;
+import java.util.function.Function;
 
+import static java.util.Collections.emptySet;
+import static java.util.Collections.singleton;
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.nullValue;
@@ -86,12 +91,16 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
         return XContentBuilder.builder(getXContentType().xContent());
     }
 
-    private XContentBuilder newXContentBuilder(String filter, boolean inclusive) throws IOException {
-        return XContentBuilder.builder(getXContentType().xContent(), new String[] { filter }, inclusive);
+    private XContentBuilder newXContentBuilderWithIncludes(String filter) throws IOException {
+        return newXContentBuilder(singleton(filter), emptySet());
     }
 
-    private XContentBuilder newXContentBuilder(String[] filters, boolean inclusive) throws IOException {
-        return XContentBuilder.builder(getXContentType().xContent(), filters, inclusive);
+    private XContentBuilder newXContentBuilderWithExcludes(String filter) throws IOException {
+        return newXContentBuilder(emptySet(), singleton(filter));
+    }
+
+    private XContentBuilder newXContentBuilder(Set<String> includes, Set<String> excludes) throws IOException {
+        return XContentBuilder.builder(getXContentType().xContent(), includes, excludes);
     }
 
     /**
@@ -173,20 +182,22 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
         return builder;
     }
 
-    /**
-     * Instanciates a new XContentBuilder with the given filters and builds a
-     * sample with it.
-     * @param inclusive
-     *            Specifies if filters are inclusive or exclusive
-     */
-    private XContentBuilder sample(String filter, boolean inclusive) throws IOException {
-        return sample(newXContentBuilder(filter, inclusive));
+    /** Create a new {@link XContentBuilder} and use it to build the sample using the given inclusive filter **/
+    private XContentBuilder sampleWithIncludes(String filter) throws IOException {
+        return sample(newXContentBuilderWithIncludes(filter));
     }
 
-    private XContentBuilder sample(String[] filters, boolean inclusive) throws IOException {
-        return sample(newXContentBuilder(filters, inclusive));
+    /** Create a new {@link XContentBuilder} and use it to build the sample using the given exclusive filter **/
+    private XContentBuilder sampleWithExcludes(String filter) throws IOException {
+        return sample(newXContentBuilderWithExcludes(filter));
     }
 
+    /** Create a new {@link XContentBuilder} and use it to build the sample using the given includes and exclusive filters **/
+    private XContentBuilder sampleWithFilters(Set<String> includes, Set<String> excludes) throws IOException {
+        return sample(newXContentBuilder(includes, excludes));
+    }
+
+    /** Create a new {@link XContentBuilder} and use it to build the sample **/
     private XContentBuilder sample() throws IOException {
         return sample(newXContentBuilder());
     }
@@ -195,23 +206,23 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
         XContentBuilder expected = sample();
 
         assertXContentBuilder(expected, sample());
-        assertXContentBuilder(expected, sample("*", true));
-        assertXContentBuilder(expected, sample("**", true));
-        assertXContentBuilder(expected, sample("xyz", false));
+        assertXContentBuilder(expected, sampleWithIncludes("*"));
+        assertXContentBuilder(expected, sampleWithIncludes("**"));
+        assertXContentBuilder(expected, sampleWithExcludes("xyz"));
     }
 
     public void testNoMatch() throws Exception {
         XContentBuilder expected = newXContentBuilder().startObject().endObject();
 
-        assertXContentBuilder(expected, sample("xyz", true));
-        assertXContentBuilder(expected, sample("*", false));
-        assertXContentBuilder(expected, sample("**", false));
+        assertXContentBuilder(expected, sampleWithIncludes("xyz"));
+        assertXContentBuilder(expected, sampleWithExcludes("*"));
+        assertXContentBuilder(expected, sampleWithExcludes("**"));
     }
 
     public void testSimpleFieldInclusive() throws Exception {
         XContentBuilder expected = newXContentBuilder().startObject().field("title", "My awesome book").endObject();
 
-        assertXContentBuilder(expected, sample("title", true));
+        assertXContentBuilder(expected, sampleWithIncludes("title"));
     }
 
     public void testSimpleFieldExclusive() throws Exception {
@@ -286,10 +297,9 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
                                                             .endObject()
                                                         .endObject();
 
-        assertXContentBuilder(expected, sample("title", false));
+        assertXContentBuilder(expected, sampleWithExcludes("title"));
     }
 
-
     public void testSimpleFieldWithWildcardInclusive() throws Exception {
         XContentBuilder expected = newXContentBuilder().startObject()
                                                             .field("price", 27.99)
@@ -343,7 +353,7 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
                                                             .endObject()
                                                         .endObject();
 
-        assertXContentBuilder(expected, sample("pr*", true));
+        assertXContentBuilder(expected, sampleWithIncludes("pr*"));
     }
 
     public void testSimpleFieldWithWildcardExclusive() throws Exception {
@@ -370,7 +380,7 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
                                                             .endArray()
                                                         .endObject();
 
-        assertXContentBuilder(expected, sample("pr*", false));
+        assertXContentBuilder(expected, sampleWithExcludes("pr*"));
     }
 
     public void testMultipleFieldsInclusive() throws Exception {
@@ -379,7 +389,7 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
                                                             .field("pages", 456)
                                                         .endObject();
 
-        assertXContentBuilder(expected, sample(new String[] { "title", "pages" }, true));
+        assertXContentBuilder(expected, sampleWithFilters(Sets.newHashSet("title", "pages"), emptySet()));
     }
 
     public void testMultipleFieldsExclusive() throws Exception {
@@ -453,10 +463,9 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
                                                             .endObject()
                                                         .endObject();
 
-        assertXContentBuilder(expected, sample(new String[] { "title", "pages" }, false));
+        assertXContentBuilder(expected, sample(newXContentBuilder(emptySet(), Sets.newHashSet("title", "pages"))));
     }
 
-
     public void testSimpleArrayInclusive() throws Exception {
         XContentBuilder expected = newXContentBuilder().startObject()
                                                             .startArray("tags")
@@ -465,7 +474,7 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
                                                             .endArray()
                                                     .endObject();
 
-        assertXContentBuilder(expected, sample("tags", true));
+        assertXContentBuilder(expected, sampleWithIncludes("tags"));
     }
 
     public void testSimpleArrayExclusive() throws Exception {
@@ -537,10 +546,9 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
                                                             .endObject()
                                                         .endObject();
 
-        assertXContentBuilder(expected, sample("tags", false));
+        assertXContentBuilder(expected, sampleWithExcludes("tags"));
     }
 
-
     public void testSimpleArrayOfObjectsInclusive() throws Exception {
         XContentBuilder expected = newXContentBuilder().startObject()
                                                             .startArray("authors")
@@ -557,9 +565,9 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
                                                             .endArray()
                                                         .endObject();
 
-        assertXContentBuilder(expected, sample("authors", true));
-        assertXContentBuilder(expected, sample("authors.*", true));
-        assertXContentBuilder(expected, sample("authors.*name", true));
+        assertXContentBuilder(expected, sampleWithIncludes("authors"));
+        assertXContentBuilder(expected, sampleWithIncludes("authors.*"));
+        assertXContentBuilder(expected, sampleWithIncludes("authors.*name"));
     }
 
     public void testSimpleArrayOfObjectsExclusive() throws Exception {
@@ -623,9 +631,9 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
                                                             .endObject()
                                                        .endObject();
 
-        assertXContentBuilder(expected, sample("authors", false));
-        assertXContentBuilder(expected, sample("authors.*", false));
-        assertXContentBuilder(expected, sample("authors.*name", false));
+        assertXContentBuilder(expected, sampleWithExcludes("authors"));
+        assertXContentBuilder(expected, sampleWithExcludes("authors.*"));
+        assertXContentBuilder(expected, sampleWithExcludes("authors.*name"));
     }
 
     public void testSimpleArrayOfObjectsPropertyInclusive() throws Exception {
@@ -640,8 +648,8 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
                                                             .endArray()
                                                         .endObject();
 
-        assertXContentBuilder(expected, sample("authors.lastname", true));
-        assertXContentBuilder(expected, sample("authors.l*", true));
+        assertXContentBuilder(expected, sampleWithIncludes("authors.lastname"));
+        assertXContentBuilder(expected, sampleWithIncludes("authors.l*"));
     }
 
     public void testSimpleArrayOfObjectsPropertyExclusive() throws Exception {
@@ -715,8 +723,8 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
                                                             .endObject()
                                                        .endObject();
 
-        assertXContentBuilder(expected, sample("authors.lastname", false));
-        assertXContentBuilder(expected, sample("authors.l*", false));
+        assertXContentBuilder(expected, sampleWithExcludes("authors.lastname"));
+        assertXContentBuilder(expected, sampleWithExcludes("authors.l*"));
     }
 
     public void testRecurseField1Inclusive() throws Exception {
@@ -768,7 +776,7 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
                                                             .endObject()
                                                         .endObject();
 
-        assertXContentBuilder(expected, sample("**.name", true));
+        assertXContentBuilder(expected, sampleWithIncludes("**.name"));
     }
 
     public void testRecurseField1Exclusive() throws Exception {
@@ -831,7 +839,7 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
                                                             .endObject()
                                                        .endObject();
 
-        assertXContentBuilder(expected, sample("**.name", false));
+        assertXContentBuilder(expected, sampleWithExcludes("**.name"));
     }
 
     public void testRecurseField2Inclusive() throws Exception {
@@ -875,7 +883,7 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
                                                             .endObject()
                                                         .endObject();
 
-        assertXContentBuilder(expected, sample("properties.**.name", true));
+        assertXContentBuilder(expected, sampleWithIncludes("properties.**.name"));
     }
 
     public void testRecurseField2Exclusive() throws Exception {
@@ -940,10 +948,9 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
                                                             .endObject()
                                                        .endObject();
 
-        assertXContentBuilder(expected, sample("properties.**.name", false));
+        assertXContentBuilder(expected, sampleWithExcludes("properties.**.name"));
     }
 
-
     public void testRecurseField3Inclusive() throws Exception {
         XContentBuilder expected = newXContentBuilder().startObject()
                                                             .startObject("properties")
@@ -970,7 +977,7 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
                                                             .endObject()
                                                         .endObject();
 
-        assertXContentBuilder(expected, sample("properties.*.en.**.name", true));
+        assertXContentBuilder(expected, sampleWithIncludes("properties.*.en.**.name"));
     }
 
     public void testRecurseField3Exclusive() throws Exception {
@@ -1040,10 +1047,9 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
                                                             .endObject()
                                                        .endObject();
 
-        assertXContentBuilder(expected, sample("properties.*.en.**.name", false));
+        assertXContentBuilder(expected, sampleWithExcludes("properties.*.en.**.name"));
     }
 
-
     public void testRecurseField4Inclusive() throws Exception {
         XContentBuilder expected = newXContentBuilder().startObject()
                                                             .startObject("properties")
@@ -1072,7 +1078,7 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
                                                             .endObject()
                                                         .endObject();
 
-        assertXContentBuilder(expected, sample("properties.**.distributors.name", true));
+        assertXContentBuilder(expected, sampleWithIncludes("properties.**.distributors.name"));
     }
 
     public void testRecurseField4Exclusive() throws Exception {
@@ -1140,7 +1146,7 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
                                                             .endObject()
                                                        .endObject();
 
-        assertXContentBuilder(expected, sample("properties.**.distributors.name", false));
+        assertXContentBuilder(expected, sampleWithExcludes("properties.**.distributors.name"));
     }
 
     public void testRawField() throws Exception {
@@ -1155,24 +1161,24 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
         // Test method: rawField(String fieldName, BytesReference content)
         assertXContentBuilder(expectedRawField, newXContentBuilder().startObject().field("foo", 0).rawField("raw", raw).endObject());
         assertXContentBuilder(expectedRawFieldFiltered,
-                newXContentBuilder("f*", true).startObject().field("foo", 0).rawField("raw", raw).endObject());
+                newXContentBuilderWithIncludes("f*").startObject().field("foo", 0).rawField("raw", raw).endObject());
         assertXContentBuilder(expectedRawFieldFiltered,
-                newXContentBuilder("r*", false).startObject().field("foo", 0).rawField("raw", raw).endObject());
+                newXContentBuilderWithExcludes("r*").startObject().field("foo", 0).rawField("raw", raw).endObject());
         assertXContentBuilder(expectedRawFieldNotFiltered,
-                newXContentBuilder("r*", true).startObject().field("foo", 0).rawField("raw", raw).endObject());
+                newXContentBuilderWithIncludes("r*").startObject().field("foo", 0).rawField("raw", raw).endObject());
         assertXContentBuilder(expectedRawFieldNotFiltered,
-                newXContentBuilder("f*", false).startObject().field("foo", 0).rawField("raw", raw).endObject());
+                newXContentBuilderWithExcludes("f*").startObject().field("foo", 0).rawField("raw", raw).endObject());
 
         // Test method: rawField(String fieldName, InputStream content)
         assertXContentBuilder(expectedRawField,
                 newXContentBuilder().startObject().field("foo", 0).rawField("raw", raw.streamInput()).endObject());
-        assertXContentBuilder(expectedRawFieldFiltered, newXContentBuilder("f*", true).startObject().field("foo", 0)
+        assertXContentBuilder(expectedRawFieldFiltered, newXContentBuilderWithIncludes("f*").startObject().field("foo", 0)
                 .rawField("raw", raw.streamInput()).endObject());
-        assertXContentBuilder(expectedRawFieldFiltered, newXContentBuilder("r*", false).startObject().field("foo", 0)
+        assertXContentBuilder(expectedRawFieldFiltered, newXContentBuilderWithExcludes("r*").startObject().field("foo", 0)
                 .rawField("raw", raw.streamInput()).endObject());
-        assertXContentBuilder(expectedRawFieldNotFiltered, newXContentBuilder("r*", true).startObject().field("foo", 0)
+        assertXContentBuilder(expectedRawFieldNotFiltered, newXContentBuilderWithIncludes("r*").startObject().field("foo", 0)
                 .rawField("raw", raw.streamInput()).endObject());
-        assertXContentBuilder(expectedRawFieldNotFiltered, newXContentBuilder("f*", false).startObject().field("foo", 0)
+        assertXContentBuilder(expectedRawFieldNotFiltered, newXContentBuilderWithExcludes("f*").startObject().field("foo", 0)
                 .rawField("raw", raw.streamInput()).endObject());
     }
 
@@ -1180,48 +1186,209 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
         // Test: Array of values (no filtering)
         XContentBuilder expected = newXContentBuilder().startObject().startArray("tags").value("lorem").value("ipsum").value("dolor")
                 .endArray().endObject();
-        assertXContentBuilder(expected, newXContentBuilder("t*", true).startObject().startArray("tags").value("lorem").value("ipsum")
-                .value("dolor").endArray().endObject());
-        assertXContentBuilder(expected, newXContentBuilder("tags", true).startObject().startArray("tags").value("lorem").value("ipsum")
+        assertXContentBuilder(expected, newXContentBuilderWithIncludes("t*").startObject().startArray("tags").value("lorem").value("ipsum")
                 .value("dolor").endArray().endObject());
-        assertXContentBuilder(expected, newXContentBuilder("a", false).startObject().startArray("tags").value("lorem").value("ipsum")
+        assertXContentBuilder(expected, newXContentBuilderWithIncludes("tags").startObject().startArray("tags").value("lorem")
+                .value("ipsum").value("dolor").endArray().endObject());
+        assertXContentBuilder(expected, newXContentBuilderWithExcludes("a").startObject().startArray("tags").value("lorem").value("ipsum")
                 .value("dolor").endArray().endObject());
 
         // Test: Array of values (with filtering)
-        assertXContentBuilder(newXContentBuilder().startObject().endObject(), newXContentBuilder("foo", true).startObject()
+        assertXContentBuilder(newXContentBuilder().startObject().endObject(), newXContentBuilderWithIncludes("foo").startObject()
                 .startArray("tags").value("lorem").value("ipsum").value("dolor").endArray().endObject());
-        assertXContentBuilder(newXContentBuilder().startObject().endObject(), newXContentBuilder("t*", false).startObject()
+        assertXContentBuilder(newXContentBuilder().startObject().endObject(), newXContentBuilderWithExcludes("t*").startObject()
                 .startArray("tags").value("lorem").value("ipsum").value("dolor").endArray().endObject());
-        assertXContentBuilder(newXContentBuilder().startObject().endObject(), newXContentBuilder("tags", false).startObject()
+        assertXContentBuilder(newXContentBuilder().startObject().endObject(), newXContentBuilderWithExcludes("tags").startObject()
                 .startArray("tags").value("lorem").value("ipsum").value("dolor").endArray().endObject());
 
         // Test: Array of objects (no filtering)
         expected = newXContentBuilder().startObject().startArray("tags").startObject().field("lastname", "lorem").endObject().startObject()
                 .field("firstname", "ipsum").endObject().endArray().endObject();
-        assertXContentBuilder(expected, newXContentBuilder("t*", true).startObject().startArray("tags").startObject()
+        assertXContentBuilder(expected, newXContentBuilderWithIncludes("t*").startObject().startArray("tags").startObject()
                 .field("lastname", "lorem").endObject().startObject().field("firstname", "ipsum").endObject().endArray().endObject());
-        assertXContentBuilder(expected, newXContentBuilder("tags", true).startObject().startArray("tags").startObject()
+        assertXContentBuilder(expected, newXContentBuilderWithIncludes("tags").startObject().startArray("tags").startObject()
                 .field("lastname", "lorem").endObject().startObject().field("firstname", "ipsum").endObject().endArray().endObject());
-        assertXContentBuilder(expected, newXContentBuilder("a", false).startObject().startArray("tags").startObject()
+        assertXContentBuilder(expected, newXContentBuilderWithExcludes("a").startObject().startArray("tags").startObject()
                 .field("lastname", "lorem").endObject().startObject().field("firstname", "ipsum").endObject().endArray().endObject());
 
         // Test: Array of objects (with filtering)
         assertXContentBuilder(newXContentBuilder().startObject().endObject(),
-                newXContentBuilder("foo", true).startObject().startArray("tags").startObject().field("lastname", "lorem").endObject()
+                newXContentBuilderWithIncludes("foo").startObject().startArray("tags").startObject().field("lastname", "lorem").endObject()
                         .startObject().field("firstname", "ipsum").endObject().endArray().endObject());
         assertXContentBuilder(newXContentBuilder().startObject().endObject(),
-                newXContentBuilder("t*", false).startObject().startArray("tags").startObject().field("lastname", "lorem").endObject()
+                newXContentBuilderWithExcludes("t*").startObject().startArray("tags").startObject().field("lastname", "lorem").endObject()
                         .startObject().field("firstname", "ipsum").endObject().endArray().endObject());
         assertXContentBuilder(newXContentBuilder().startObject().endObject(),
-                newXContentBuilder("tags", false).startObject().startArray("tags").startObject().field("lastname", "lorem").endObject()
+                newXContentBuilderWithExcludes("tags").startObject().startArray("tags").startObject().field("lastname", "lorem").endObject()
                         .startObject().field("firstname", "ipsum").endObject().endArray().endObject());
 
         // Test: Array of objects (with partial filtering)
         expected = newXContentBuilder().startObject().startArray("tags").startObject().field("firstname", "ipsum").endObject().endArray()
                 .endObject();
-        assertXContentBuilder(expected, newXContentBuilder("t*.firstname", true).startObject().startArray("tags").startObject()
+        assertXContentBuilder(expected, newXContentBuilderWithIncludes("t*.firstname").startObject().startArray("tags").startObject()
                 .field("lastname", "lorem").endObject().startObject().field("firstname", "ipsum").endObject().endArray().endObject());
-        assertXContentBuilder(expected, newXContentBuilder("t*.lastname", false).startObject().startArray("tags").startObject()
+        assertXContentBuilder(expected, newXContentBuilderWithExcludes("t*.lastname").startObject().startArray("tags").startObject()
                 .field("lastname", "lorem").endObject().startObject().field("firstname", "ipsum").endObject().endArray().endObject());
     }
+
+    public void testEmptyObject() throws IOException {
+        final Function<XContentBuilder, XContentBuilder> build = builder -> {
+            try {
+                return builder.startObject().startObject("foo").endObject().endObject();
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        };
+
+        XContentBuilder expected = build.apply(newXContentBuilder());
+        assertXContentBuilder(expected, build.apply(newXContentBuilderWithIncludes("foo")));
+        assertXContentBuilder(expected, build.apply(newXContentBuilderWithExcludes("bar")));
+        assertXContentBuilder(expected, build.apply(newXContentBuilder(singleton("f*"), singleton("baz"))));
+
+        expected = newXContentBuilder().startObject().endObject();
+        assertXContentBuilder(expected, build.apply(newXContentBuilderWithExcludes("foo")));
+        assertXContentBuilder(expected, build.apply(newXContentBuilderWithIncludes("bar")));
+        assertXContentBuilder(expected, build.apply(newXContentBuilder(singleton("f*"), singleton("foo"))));
+    }
+
+    public void testSingleFieldObject() throws IOException {
+        final Function<XContentBuilder, XContentBuilder> build = builder -> {
+            try {
+                return builder.startObject().startObject("foo").field("bar", "test").endObject().endObject();
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        };
+
+        XContentBuilder expected = build.apply(newXContentBuilder());
+        assertXContentBuilder(expected, build.apply(newXContentBuilderWithIncludes("foo.bar")));
+        assertXContentBuilder(expected, build.apply(newXContentBuilderWithExcludes("foo.baz")));
+        assertXContentBuilder(expected, build.apply(newXContentBuilder(singleton("foo"), singleton("foo.baz"))));
+
+        expected = newXContentBuilder().startObject().endObject();
+        assertXContentBuilder(expected, build.apply(newXContentBuilderWithExcludes("foo.bar")));
+        assertXContentBuilder(expected, build.apply(newXContentBuilder(singleton("foo"), singleton("foo.b*"))));
+    }
+
+    public void testSingleFieldWithBothExcludesIncludes() throws IOException {
+        XContentBuilder expected = newXContentBuilder()
+            .startObject()
+                .field("pages", 456)
+                .field("price", 27.99)
+            .endObject();
+
+        assertXContentBuilder(expected, sampleWithFilters(singleton("p*"), singleton("properties")));
+    }
+
+    public void testObjectsInArrayWithBothExcludesIncludes() throws IOException {
+        Set<String> includes = Sets.newHashSet("tags", "authors");
+        Set<String> excludes = singleton("authors.name");
+
+        XContentBuilder expected = newXContentBuilder()
+            .startObject()
+                .startArray("tags")
+                    .value("elasticsearch")
+                    .value("java")
+                .endArray()
+                .startArray("authors")
+                    .startObject()
+                        .field("lastname", "John")
+                        .field("firstname", "Doe")
+                    .endObject()
+                    .startObject()
+                        .field("lastname", "William")
+                        .field("firstname", "Smith")
+                    .endObject()
+                .endArray()
+            .endObject();
+
+        assertXContentBuilder(expected, sampleWithFilters(includes, excludes));
+    }
+
+    public void testRecursiveObjectsInArrayWithBothExcludesIncludes() throws IOException {
+        Set<String> includes = Sets.newHashSet("**.language", "properties.weight");
+        Set<String> excludes = singleton("**.distributors");
+
+        XContentBuilder expected = newXContentBuilder()
+            .startObject()
+                .startObject("properties")
+                    .field("weight", 0.8d)
+                    .startObject("language")
+                        .startObject("en")
+                            .field("lang", "English")
+                            .field("available", true)
+                        .endObject()
+                        .startObject("fr")
+                            .field("lang", "French")
+                            .field("available", false)
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject();
+
+        assertXContentBuilder(expected, sampleWithFilters(includes, excludes));
+    }
+
+    public void testRecursiveSameObjectWithBothExcludesIncludes() throws IOException {
+        Set<String> includes = singleton("**.distributors");
+        Set<String> excludes = singleton("**.distributors");
+
+        XContentBuilder expected = newXContentBuilder().startObject().endObject();
+        assertXContentBuilder(expected, sampleWithFilters(includes, excludes));
+    }
+
+    public void testRecursiveObjectsPropertiesWithBothExcludesIncludes() throws IOException {
+        Set<String> includes = singleton("**.en.*");
+        Set<String> excludes = Sets.newHashSet("**.distributors.*.name", "**.street");
+
+        XContentBuilder expected = newXContentBuilder()
+            .startObject()
+                .startObject("properties")
+                    .startObject("language")
+                        .startObject("en")
+                            .field("lang", "English")
+                            .field("available", true)
+                            .startArray("distributors")
+                                .startObject()
+                                    .field("name", "The Book Shop")
+                                    .startArray("addresses")
+                                        .startObject()
+                                            .field("city", "London")
+                                        .endObject()
+                                        .startObject()
+                                            .field("city", "Stornoway")
+                                        .endObject()
+                                    .endArray()
+                                .endObject()
+                                .startObject()
+                                    .field("name", "Sussex Books House")
+                                .endObject()
+                            .endArray()
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject();
+
+        assertXContentBuilder(expected, sampleWithFilters(includes, excludes));
+    }
+
+    public void testWithLfAtEnd() throws IOException {
+        final Function<XContentBuilder, XContentBuilder> build = builder -> {
+            try {
+                return builder.startObject().startObject("foo").field("bar", "baz").endObject().endObject().prettyPrint().lfAtEnd();
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        };
+
+        XContentBuilder expected = build.apply(newXContentBuilder());
+        assertXContentBuilder(expected, build.apply(newXContentBuilderWithIncludes("foo")));
+        assertXContentBuilder(expected, build.apply(newXContentBuilderWithExcludes("bar")));
+        assertXContentBuilder(expected, build.apply(newXContentBuilder(singleton("f*"), singleton("baz"))));
+
+        expected = newXContentBuilder().startObject().endObject().prettyPrint().lfAtEnd();
+        assertXContentBuilder(expected, build.apply(newXContentBuilderWithExcludes("foo")));
+        assertXContentBuilder(expected, build.apply(newXContentBuilderWithIncludes("bar")));
+        assertXContentBuilder(expected, build.apply(newXContentBuilder(singleton("f*"), singleton("foo"))));
+    }
 }

core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathGeneratorFilteringTests.java

@@ -25,6 +25,8 @@ import com.fasterxml.jackson.core.filter.FilteringGeneratorDelegate;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.test.ESTestCase;
 
+import java.util.Collections;
+
 import static org.hamcrest.Matchers.equalTo;
 
 public class FilterPathGeneratorFilteringTests extends ESTestCase {
@@ -135,7 +137,7 @@ public class FilterPathGeneratorFilteringTests extends ESTestCase {
     private void assertResult(String input, String filter, boolean inclusive, String expected) throws Exception {
         try (BytesStreamOutput os = new BytesStreamOutput()) {
             try (FilteringGeneratorDelegate generator = new FilteringGeneratorDelegate(JSON_FACTORY.createGenerator(os),
-                    new FilterPathBasedFilter(new String[] { filter }, inclusive), true, true)) {
+                    new FilterPathBasedFilter(Collections.singleton(filter), inclusive), true, true)) {
                 try (JsonParser parser = JSON_FACTORY.createParser(replaceQuotes(input))) {
                     while (parser.nextToken() != null) {
                         generator.copyCurrentStructure(parser);

core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathTests.java

@@ -19,8 +19,12 @@
 
 package org.elasticsearch.common.xcontent.support.filtering;
 
+import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.test.ESTestCase;
 
+import java.util.Set;
+
+import static java.util.Collections.singleton;
 import static org.hamcrest.Matchers.arrayWithSize;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
@@ -33,7 +37,7 @@ public class FilterPathTests extends ESTestCase {
     public void testSimpleFilterPath() {
         final String input = "test";
 
-        FilterPath[] filterPaths = FilterPath.compile(input);
+        FilterPath[] filterPaths = FilterPath.compile(singleton(input));
         assertNotNull(filterPaths);
         assertThat(filterPaths, arrayWithSize(1));
 
@@ -52,7 +56,7 @@ public class FilterPathTests extends ESTestCase {
     public void testFilterPathWithSubField() {
         final String input = "foo.bar";
 
-        FilterPath[] filterPaths = FilterPath.compile(input);
+        FilterPath[] filterPaths = FilterPath.compile(singleton(input));
         assertNotNull(filterPaths);
         assertThat(filterPaths, arrayWithSize(1));
 
@@ -76,7 +80,7 @@ public class FilterPathTests extends ESTestCase {
     public void testFilterPathWithSubFields() {
         final String input = "foo.bar.quz";
 
-        FilterPath[] filterPaths = FilterPath.compile(input);
+        FilterPath[] filterPaths = FilterPath.compile(singleton(input));
         assertNotNull(filterPaths);
         assertThat(filterPaths, arrayWithSize(1));
 
@@ -103,13 +107,13 @@ public class FilterPathTests extends ESTestCase {
     }
 
     public void testEmptyFilterPath() {
-        FilterPath[] filterPaths = FilterPath.compile("");
+        FilterPath[] filterPaths = FilterPath.compile(singleton(""));
         assertNotNull(filterPaths);
         assertThat(filterPaths, arrayWithSize(0));
     }
 
     public void testNullFilterPath() {
-        FilterPath[] filterPaths = FilterPath.compile((String) null);
+        FilterPath[] filterPaths = FilterPath.compile(singleton(null));
         assertNotNull(filterPaths);
         assertThat(filterPaths, arrayWithSize(0));
     }
@@ -117,7 +121,7 @@ public class FilterPathTests extends ESTestCase {
     public void testFilterPathWithEscapedDots() {
         String input = "w.0.0.t";
 
-        FilterPath[] filterPaths = FilterPath.compile(input);
+        FilterPath[] filterPaths = FilterPath.compile(singleton(input));
         assertNotNull(filterPaths);
         assertThat(filterPaths, arrayWithSize(1));
 
@@ -149,7 +153,7 @@ public class FilterPathTests extends ESTestCase {
 
         input = "w\\.0\\.0\\.t";
 
-        filterPaths = FilterPath.compile(input);
+        filterPaths = FilterPath.compile(singleton(input));
         assertNotNull(filterPaths);
         assertThat(filterPaths, arrayWithSize(1));
 
@@ -167,7 +171,7 @@ public class FilterPathTests extends ESTestCase {
 
         input = "w\\.0.0\\.t";
 
-        filterPaths = FilterPath.compile(input);
+        filterPaths = FilterPath.compile(singleton(input));
         assertNotNull(filterPaths);
         assertThat(filterPaths, arrayWithSize(1));
 
@@ -188,7 +192,7 @@ public class FilterPathTests extends ESTestCase {
     }
 
     public void testSimpleWildcardFilterPath() {
-        FilterPath[] filterPaths = FilterPath.compile("*");
+        FilterPath[] filterPaths = FilterPath.compile(singleton("*"));
         assertNotNull(filterPaths);
         assertThat(filterPaths, arrayWithSize(1));
 
@@ -206,7 +210,7 @@ public class FilterPathTests extends ESTestCase {
     public void testWildcardInNameFilterPath() {
         String input = "f*o.bar";
 
-        FilterPath[] filterPaths = FilterPath.compile(input);
+        FilterPath[] filterPaths = FilterPath.compile(singleton(input));
         assertNotNull(filterPaths);
         assertThat(filterPaths, arrayWithSize(1));
 
@@ -232,7 +236,7 @@ public class FilterPathTests extends ESTestCase {
     }
 
     public void testDoubleWildcardFilterPath() {
-        FilterPath[] filterPaths = FilterPath.compile("**");
+        FilterPath[] filterPaths = FilterPath.compile(singleton("**"));
         assertNotNull(filterPaths);
         assertThat(filterPaths, arrayWithSize(1));
 
@@ -250,7 +254,7 @@ public class FilterPathTests extends ESTestCase {
     public void testStartsWithDoubleWildcardFilterPath() {
         String input = "**.bar";
 
-        FilterPath[] filterPaths = FilterPath.compile(input);
+        FilterPath[] filterPaths = FilterPath.compile(singleton(input));
         assertNotNull(filterPaths);
         assertThat(filterPaths, arrayWithSize(1));
 
@@ -274,7 +278,7 @@ public class FilterPathTests extends ESTestCase {
     public void testContainsDoubleWildcardFilterPath() {
         String input = "foo.**.bar";
 
-        FilterPath[] filterPaths = FilterPath.compile(input);
+        FilterPath[] filterPaths = FilterPath.compile(singleton(input));
         assertNotNull(filterPaths);
         assertThat(filterPaths, arrayWithSize(1));
 
@@ -302,7 +306,7 @@ public class FilterPathTests extends ESTestCase {
     }
 
     public void testMultipleFilterPaths() {
-        String[] inputs = {"foo.**.bar.*", "test.dot\\.ted"};
+        Set<String> inputs = Sets.newHashSet("foo.**.bar.*", "test.dot\\.ted");
 
         FilterPath[] filterPaths = FilterPath.compile(inputs);
         assertNotNull(filterPaths);

docs/reference/api-conventions.asciidoc

@@ -276,6 +276,41 @@ curl 'localhost:9200/_segments?pretty&filter_path=indices.**.version'
 }
 --------------------------------------------------
 
+It is also possible to exclude one or more fields by prefixing the filter with the char `-`:
+
+[source,sh]
+--------------------------------------------------
+curl -XGET 'localhost:9200/_count?filter_path=-_shards'
+{
+  "count" : 1
+}
+%
+--------------------------------------------------
+
+And for more control, both inclusive and exclusive filters can be combined in the same expression. In
+this case, the exclusive filters will be applied first and the result will be filtered again using the
+inclusive filters:
+
+[source,sh]
+--------------------------------------------------
+curl -XGET 'localhost:9200/_cluster/state?filter_path=metadata.indices.*.state,-metadata.indices.logs-*'
+{
+  "metadata" : {
+    "indices" : {
+      "index-1" : {
+        "state" : "open"
+      },
+      "index-3" : {
+        "state" : "open"
+      },
+      "index-2" : {
+        "state" : "open"
+      }
+    }
+  }
+}%
+--------------------------------------------------
+
 Note that elasticsearch sometimes returns directly the raw value of a field,
 like the `_source` field. If you want to filter `_source` fields, you should
 consider combining the already existing `_source` parameter (see

rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/20_response_filtering.yaml

@@ -152,3 +152,49 @@
   - is_false: nodes.$master.fs.data.0.path
   - is_true:  nodes.$master.fs.data.0.type
   - is_true:  nodes.$master.fs.data.0.total_in_bytes
+
+---
+"Nodes Stats filtered using both includes and excludes filters":
+  - do:
+      cluster.state: {}
+
+  # Get master node id
+  - set: { master_node: master }
+
+  # Nodes Stats with "nodes" field but no JVM stats
+  - do:
+      nodes.stats:
+        filter_path: [ "nodes", "-nodes.*.jvm", "-nodes.*.indices" ]
+
+  - is_false: cluster_name
+  - is_true:  nodes
+  - is_true:  nodes.$master.name
+  - is_true:  nodes.$master.os
+  - is_false: nodes.$master.indices
+  - is_false: nodes.$master.jvm
+
+  # Nodes Stats with "nodes.*.indices" field and sub-fields but no indices segments
+  - do:
+      nodes.stats:
+        filter_path: "nodes.*.indices,-nodes.*.indices.segments"
+
+  - is_false: cluster_name
+  - is_true:  nodes
+  - is_false: nodes.$master.name
+  - is_true:  nodes.$master.indices
+  - is_true:  nodes.$master.indices.docs
+  - is_false: nodes.$master.indices.segments
+
+  # Nodes Stats with "nodes.*.fs.data.t*" fields but no "type" field
+  - do:
+      nodes.stats:
+        filter_path: "nodes.*.fs.data.t*,-**.type"
+
+  - is_false: cluster_name
+  - is_true:  nodes
+  - is_false: nodes.$master.name
+  - is_false: nodes.$master.indices
+  - is_false: nodes.$master.jvm
+  - is_true:  nodes.$master.fs.data
+  - is_false: nodes.$master.fs.data.0.type
+  - is_true:  nodes.$master.fs.data.0.total_in_bytes

rest-api-spec/src/main/resources/rest-api-spec/test/search/70_response_filtering.yaml

@@ -84,3 +84,64 @@
   - is_true:  hits.hits.1._index
   - is_false: hits.hits.1._type
   - is_true:  hits.hits.1._id
+
+---
+"Search results filtered using both includes and excludes filters":
+  - do:
+      bulk:
+        refresh: true
+        body: |
+          {"index": {"_index": "index-1", "_type": "type-1", "_id": "1"}}
+          {"name": "First document", "properties": {"size": 123, "meta": {"foo": "bar"}}}
+          {"index": {"_index": "index-1", "_type": "type-1", "_id": "2"}}
+          {"name": "Second document", "properties": {"size": 465, "meta": {"foo": "bar", "baz": "qux"}}}
+
+  - do:
+      search:
+        filter_path: [  "-**._source.properties", "**._source" ]
+        body: { query: { match_all: {} } }
+
+  - is_false: took
+  - is_true:  hits.hits.0._source
+  - is_true:  hits.hits.0._source.name
+  - is_false: hits.hits.0._source.properties
+  - is_true:  hits.hits.1._source
+  - is_true:  hits.hits.1._source.name
+  - is_false: hits.hits.1._source.properties
+
+  - do:
+      search:
+        filter_path: [ "**.properties" , "-hits.hits._source.properties.meta" ]
+        body: { query: { match_all: {} } }
+
+  - is_false: took
+  - is_true:  hits.hits.0._source
+  - is_false: hits.hits.0._source.name
+  - is_true:  hits.hits.0._source.properties
+  - is_true:  hits.hits.0._source.properties.size
+  - is_false: hits.hits.0._source.properties.meta
+  - is_true:  hits.hits.1._source
+  - is_false: hits.hits.1._source.name
+  - is_true:  hits.hits.1._source.properties
+  - is_true:  hits.hits.1._source.properties.size
+  - is_false: hits.hits.1._source.properties.meta
+
+  - do:
+      search:
+        filter_path: "**._source,-**.meta.foo"
+        body: { query: { match_all: {} } }
+
+  - is_false: took
+  - is_true:  hits.hits.0._source
+  - is_true:  hits.hits.0._source.name
+  - is_true:  hits.hits.0._source.properties
+  - is_true:  hits.hits.0._source.properties.size
+  - is_false: hits.hits.0._source.properties.meta.foo
+
+  - do:
+      count:
+        filter_path: "-*"
+        body: { query: { match_all: {} } }
+
+  - is_false: count
+  - is_false: _shards