[Test] Use a common testing class for all XContent filtering tests (#25491)

We have two ways to filter XContent:

- The first method is to parse the XContent as a map and use
XContentMapValues.filter(). This method filters the content of the map
using an automaton. It is used for source filtering, both at search and
indexing time. It performs well but can generate a lot of objects and
garbage collections when large XContent are filtered. It also returns
empty objects (see f2710c16ebd918f646be9d0ab64b4871c25be4c2) when all
the sub fields have been filtered out and handle dots in field names as
if they were sub fields.

- The second method is to parse the XContent and copy the XContentParser
 structure to a XContentBuilder initialized with includes/excludes
 filters. This method uses the Jackson streaming filter feature. It is
 used by the Response Filtering ('filter_path') feature. It does not
 generate a lot of objects, and does not return empty objects and also
 does not handle dots in field names explicitely.

 Both methods have similar goals but different tests. This commit changes
 the current XContentBuilder test class so that it becomes a more generic
 testing class and we can now ensure that filtering methods generate the
 same results.

 It also removes some tests from the XContentMapValuesTests class that
 should be in XContentParserTests.

core/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java

@@ -26,9 +26,12 @@ import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.singletonMap;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -193,4 +196,84 @@ public class XContentParserTests extends ESTestCase {
             assertTrue(parser.booleanValue());
         }
     }
+
+    public void testEmptyList() throws IOException {
+        XContentBuilder builder = XContentFactory.jsonBuilder().startObject()
+                .startArray("some_array")
+                .endArray().endObject();
+
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, builder.string())) {
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
+            assertEquals("some_array", parser.currentName());
+            if (random().nextBoolean()) {
+                // sometimes read the start array token, sometimes not
+                assertEquals(XContentParser.Token.START_ARRAY, parser.nextToken());
+            }
+            assertEquals(Collections.emptyList(), parser.list());
+        }
+    }
+
+    public void testSimpleList() throws IOException {
+        XContentBuilder builder = XContentFactory.jsonBuilder().startObject()
+                .startArray("some_array")
+                .value(1)
+                .value(3)
+                .value(0)
+                .endArray().endObject();
+
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, builder.string())) {
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
+            assertEquals("some_array", parser.currentName());
+            if (random().nextBoolean()) {
+                // sometimes read the start array token, sometimes not
+                assertEquals(XContentParser.Token.START_ARRAY, parser.nextToken());
+            }
+            assertEquals(Arrays.asList(1, 3, 0), parser.list());
+        }
+    }
+
+    public void testNestedList() throws IOException {
+        XContentBuilder builder = XContentFactory.jsonBuilder().startObject()
+                .startArray("some_array")
+                .startArray().endArray()
+                .startArray().value(1).value(3).endArray()
+                .startArray().value(2).endArray()
+                .endArray().endObject();
+
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, builder.string())) {
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
+            assertEquals("some_array", parser.currentName());
+            if (random().nextBoolean()) {
+                // sometimes read the start array token, sometimes not
+                assertEquals(XContentParser.Token.START_ARRAY, parser.nextToken());
+            }
+            assertEquals(
+                    Arrays.asList(Collections.<Integer>emptyList(), Arrays.asList(1, 3), Arrays.asList(2)),
+                    parser.list());
+        }
+    }
+
+    public void testNestedMapInList() throws IOException {
+        XContentBuilder builder = XContentFactory.jsonBuilder().startObject()
+                .startArray("some_array")
+                .startObject().field("foo", "bar").endObject()
+                .startObject().endObject()
+                .endArray().endObject();
+
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, builder.string())) {
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
+            assertEquals("some_array", parser.currentName());
+            if (random().nextBoolean()) {
+                // sometimes read the start array token, sometimes not
+                assertEquals(XContentParser.Token.START_ARRAY, parser.nextToken());
+            }
+            assertEquals(
+                    Arrays.asList(singletonMap("foo", "bar"), emptyMap()),
+                    parser.list());
+        }
+    }
 }

core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractFilteringJsonGeneratorTestCase.java → core/src/test/java/org/elasticsearch/common/xcontent/support/AbstractFilteringTestCase.java

@@ -17,99 +17,35 @@
  * under the License.
  */
 
-package org.elasticsearch.common.xcontent.support.filtering;
+package org.elasticsearch.common.xcontent.support;
 
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 import java.util.Set;
-import java.util.function.Function;
 
 import static java.util.Collections.emptySet;
 import static java.util.Collections.singleton;
-import static org.hamcrest.CoreMatchers.is;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.nullValue;
 
-public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase {
-
-    protected abstract XContentType getXContentType();
-
-    protected abstract void assertXContentBuilder(XContentBuilder expected, XContentBuilder builder);
-
-    protected void assertString(XContentBuilder expected, XContentBuilder builder) {
-        assertNotNull(builder);
-        assertNotNull(expected);
-
-        // Verify that the result is equal to the expected string
-        assertThat(builder.bytes().utf8ToString(), is(expected.bytes().utf8ToString()));
-    }
-
-    protected void assertBinary(XContentBuilder expected, XContentBuilder builder) {
-        assertNotNull(builder);
-        assertNotNull(expected);
-
-        try {
-            XContent xContent = XContentFactory.xContent(builder.contentType());
-            XContentParser jsonParser = createParser(xContent, expected.bytes());
-            XContentParser testParser = createParser(xContent, builder.bytes());
-
-            while (true) {
-                XContentParser.Token token1 = jsonParser.nextToken();
-                XContentParser.Token token2 = testParser.nextToken();
-                if (token1 == null) {
-                    assertThat(token2, nullValue());
-                    return;
-                }
-                assertThat(token1, equalTo(token2));
-                switch (token1) {
-                    case FIELD_NAME:
-                        assertThat(jsonParser.currentName(), equalTo(testParser.currentName()));
-                        break;
-                    case VALUE_STRING:
-                        assertThat(jsonParser.text(), equalTo(testParser.text()));
-                        break;
-                    case VALUE_NUMBER:
-                        assertThat(jsonParser.numberType(), equalTo(testParser.numberType()));
-                        assertThat(jsonParser.numberValue(), equalTo(testParser.numberValue()));
-                        break;
-                }
-            }
-        } catch (Exception e) {
-            fail("Fail to verify the result of the XContentBuilder: " + e.getMessage());
-        }
-    }
-
-    private XContentBuilder newXContentBuilder() throws IOException {
-        return XContentBuilder.builder(getXContentType().xContent());
-    }
+/**
+ * Tests for {@link XContent} filtering.
+ */
+public abstract class AbstractFilteringTestCase extends ESTestCase {
 
-    private XContentBuilder newXContentBuilderWithIncludes(String filter) throws IOException {
-        return newXContentBuilder(singleton(filter), emptySet());
+    @FunctionalInterface
+    protected interface Builder extends CheckedFunction<XContentBuilder, XContentBuilder, IOException> {
     }
 
-    private XContentBuilder newXContentBuilderWithExcludes(String filter) throws IOException {
-        return newXContentBuilder(emptySet(), singleton(filter));
-    }
+    protected abstract void testFilter(Builder expected, Builder actual, Set<String> includes, Set<String> excludes) throws IOException;
 
-    private XContentBuilder newXContentBuilder(Set<String> includes, Set<String> excludes) throws IOException {
-        return XContentBuilder.builder(getXContentType().xContent(), includes, excludes);
-    }
-
-    /**
-     * Build a sample using a given XContentBuilder
-     */
-    private XContentBuilder sample(XContentBuilder builder) throws IOException {
-        assertNotNull(builder);
-        builder.startObject()
-                .field("title", "My awesome book")
+    /** Sample test case **/
+    private static final Builder SAMPLE = builder -> builder.startObject()
+            .field("title", "My awesome book")
                 .field("pages", 456)
                 .field("price", 27.99)
                 .field("timestamp", 1428582942867L)
@@ -179,54 +115,32 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
                     .endObject()
                 .endObject()
             .endObject();
-        return builder;
-    }
-
-    /** Create a new {@link XContentBuilder} and use it to build the sample using the given inclusive filter **/
-    private XContentBuilder sampleWithIncludes(String filter) throws IOException {
-        return sample(newXContentBuilderWithIncludes(filter));
-    }
-
-    /** Create a new {@link XContentBuilder} and use it to build the sample using the given exclusive filter **/
-    private XContentBuilder sampleWithExcludes(String filter) throws IOException {
-        return sample(newXContentBuilderWithExcludes(filter));
-    }
-
-    /** Create a new {@link XContentBuilder} and use it to build the sample using the given includes and exclusive filters **/
-    private XContentBuilder sampleWithFilters(Set<String> includes, Set<String> excludes) throws IOException {
-        return sample(newXContentBuilder(includes, excludes));
-    }
-
-    /** Create a new {@link XContentBuilder} and use it to build the sample **/
-    private XContentBuilder sample() throws IOException {
-        return sample(newXContentBuilder());
-    }
 
     public void testNoFiltering() throws Exception {
-        XContentBuilder expected = sample();
+        final Builder expected = SAMPLE;
 
-        assertXContentBuilder(expected, sample());
-        assertXContentBuilder(expected, sampleWithIncludes("*"));
-        assertXContentBuilder(expected, sampleWithIncludes("**"));
-        assertXContentBuilder(expected, sampleWithExcludes("xyz"));
+        testFilter(expected, SAMPLE, emptySet(), emptySet());
+        testFilter(expected, SAMPLE, singleton("*"), emptySet());
+        testFilter(expected, SAMPLE, singleton("**"), emptySet());
+        testFilter(expected, SAMPLE, emptySet(), singleton("xyz"));
     }
 
     public void testNoMatch() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject().endObject();
+        final Builder expected = builder -> builder.startObject().endObject();
 
-        assertXContentBuilder(expected, sampleWithIncludes("xyz"));
-        assertXContentBuilder(expected, sampleWithExcludes("*"));
-        assertXContentBuilder(expected, sampleWithExcludes("**"));
+        testFilter(expected, SAMPLE, singleton("xyz"), emptySet());
+        testFilter(expected, SAMPLE, emptySet(), singleton("*"));
+        testFilter(expected, SAMPLE, emptySet(), singleton("**"));
     }
 
     public void testSimpleFieldInclusive() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject().field("title", "My awesome book").endObject();
+        final Builder expected = builder -> builder.startObject().field("title", "My awesome book").endObject();
 
-        assertXContentBuilder(expected, sampleWithIncludes("title"));
+        testFilter(expected, SAMPLE, singleton("title"), emptySet());
     }
 
     public void testSimpleFieldExclusive() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject()
+        final Builder expected = builder -> builder.startObject()
                                                             .field("pages", 456)
                                                             .field("price", 27.99)
                                                             .field("timestamp", 1428582942867L)
@@ -297,11 +211,11 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
                                                             .endObject()
                                                         .endObject();
 
-        assertXContentBuilder(expected, sampleWithExcludes("title"));
+        testFilter(expected, SAMPLE, emptySet(), singleton("title"));
     }
 
     public void testSimpleFieldWithWildcardInclusive() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject()
+        final Builder expected = builder -> builder.startObject()
                                                             .field("price", 27.99)
                                                             .startObject("properties")
                                                                 .field("weight", 0.8d)
@@ -353,11 +267,11 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
                                                             .endObject()
                                                         .endObject();
 
-        assertXContentBuilder(expected, sampleWithIncludes("pr*"));
+        testFilter(expected, SAMPLE, singleton("pr*"), emptySet());
     }
 
     public void testSimpleFieldWithWildcardExclusive() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject()
+        final Builder expected = builder -> builder.startObject()
                                                             .field("title", "My awesome book")
                                                             .field("pages", 456)
                                                             .field("timestamp", 1428582942867L)
@@ -380,20 +294,20 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
                                                             .endArray()
                                                         .endObject();
 
-        assertXContentBuilder(expected, sampleWithExcludes("pr*"));
+        testFilter(expected, SAMPLE, emptySet(), singleton("pr*"));
     }
 
     public void testMultipleFieldsInclusive() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject()
+        final Builder expected = builder -> builder.startObject()
                                                             .field("title", "My awesome book")
                                                             .field("pages", 456)
                                                         .endObject();
 
-        assertXContentBuilder(expected, sampleWithFilters(Sets.newHashSet("title", "pages"), emptySet()));
+        testFilter(expected, SAMPLE, Sets.newHashSet("title", "pages"), emptySet());
     }
 
     public void testMultipleFieldsExclusive() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject()
+        final Builder expected = builder -> builder.startObject()
                                                             .field("price", 27.99)
                                                             .field("timestamp", 1428582942867L)
                                                             .nullField("default")
@@ -463,22 +377,22 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
                                                             .endObject()
                                                         .endObject();
 
-        assertXContentBuilder(expected, sample(newXContentBuilder(emptySet(), Sets.newHashSet("title", "pages"))));
+        testFilter(expected, SAMPLE, emptySet(), Sets.newHashSet("title", "pages"));
     }
 
     public void testSimpleArrayInclusive() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject()
+        final Builder expected = builder -> builder.startObject()
                                                             .startArray("tags")
                                                                 .value("elasticsearch")
                                                                 .value("java")
                                                             .endArray()
                                                     .endObject();
 
-        assertXContentBuilder(expected, sampleWithIncludes("tags"));
+        testFilter(expected, SAMPLE, singleton("tags"), emptySet());
     }
 
     public void testSimpleArrayExclusive() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject()
+        final Builder expected = builder -> builder.startObject()
                                                             .field("title", "My awesome book")
                                                             .field("pages", 456)
                                                             .field("price", 27.99)
@@ -546,11 +460,11 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
                                                             .endObject()
                                                         .endObject();
 
-        assertXContentBuilder(expected, sampleWithExcludes("tags"));
+        testFilter(expected, SAMPLE, emptySet(), singleton("tags"));
     }
 
     public void testSimpleArrayOfObjectsInclusive() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject()
+        final Builder expected = builder -> builder.startObject()
                                                             .startArray("authors")
                                                                 .startObject()
                                                                     .field("name", "John Doe")
@@ -565,13 +479,13 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
                                                             .endArray()
                                                         .endObject();
 
-        assertXContentBuilder(expected, sampleWithIncludes("authors"));
-        assertXContentBuilder(expected, sampleWithIncludes("authors.*"));
-        assertXContentBuilder(expected, sampleWithIncludes("authors.*name"));
+        testFilter(expected, SAMPLE, singleton("authors"), emptySet());
+        testFilter(expected, SAMPLE, singleton("authors.*"), emptySet());
+        testFilter(expected, SAMPLE, singleton("authors.*name"), emptySet());
     }
 
     public void testSimpleArrayOfObjectsExclusive() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject()
+        final Builder expected = builder -> builder.startObject()
                                                             .field("title", "My awesome book")
                                                             .field("pages", 456)
                                                             .field("price", 27.99)
@@ -631,13 +545,13 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
                                                             .endObject()
                                                        .endObject();
 
-        assertXContentBuilder(expected, sampleWithExcludes("authors"));
-        assertXContentBuilder(expected, sampleWithExcludes("authors.*"));
-        assertXContentBuilder(expected, sampleWithExcludes("authors.*name"));
+        testFilter(expected, SAMPLE, emptySet(), singleton("authors"));
+        testFilter(expected, SAMPLE, emptySet(), singleton("authors.*"));
+        testFilter(expected, SAMPLE, emptySet(), singleton("authors.*name"));
     }
 
     public void testSimpleArrayOfObjectsPropertyInclusive() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject()
+        final Builder expected = builder -> builder.startObject()
                                                             .startArray("authors")
                                                                 .startObject()
                                                                     .field("lastname", "John")
@@ -648,12 +562,12 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
                                                             .endArray()
                                                         .endObject();
 
-        assertXContentBuilder(expected, sampleWithIncludes("authors.lastname"));
-        assertXContentBuilder(expected, sampleWithIncludes("authors.l*"));
+        testFilter(expected, SAMPLE, singleton("authors.lastname"), emptySet());
+        testFilter(expected, SAMPLE, singleton("authors.l*"), emptySet());
     }
 
     public void testSimpleArrayOfObjectsPropertyExclusive() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject()
+        final Builder expected = builder -> builder.startObject()
                                                             .field("title", "My awesome book")
                                                             .field("pages", 456)
                                                             .field("price", 27.99)
@@ -723,12 +637,12 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
                                                             .endObject()
                                                        .endObject();
 
-        assertXContentBuilder(expected, sampleWithExcludes("authors.lastname"));
-        assertXContentBuilder(expected, sampleWithExcludes("authors.l*"));
+        testFilter(expected, SAMPLE, emptySet(), singleton("authors.lastname"));
+        testFilter(expected, SAMPLE, emptySet(), singleton("authors.l*"));
     }
 
     public void testRecurseField1Inclusive() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject()
+        final Builder expected = builder -> builder.startObject()
                                                             .startArray("authors")
                                                                 .startObject()
                                                                     .field("name", "John Doe")
@@ -776,11 +690,11 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
                                                             .endObject()
                                                         .endObject();
 
-        assertXContentBuilder(expected, sampleWithIncludes("**.name"));
+        testFilter(expected, SAMPLE, singleton("**.name"), emptySet());
     }
 
     public void testRecurseField1Exclusive() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject()
+        final Builder expected = builder -> builder.startObject()
                                                             .field("title", "My awesome book")
                                                             .field("pages", 456)
                                                             .field("price", 27.99)
@@ -839,11 +753,11 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
                                                             .endObject()
                                                        .endObject();
 
-        assertXContentBuilder(expected, sampleWithExcludes("**.name"));
+        testFilter(expected, SAMPLE, emptySet(), singleton("**.name"));
     }
 
     public void testRecurseField2Inclusive() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject()
+        final Builder expected = builder -> builder.startObject()
                                                             .startObject("properties")
                                                                 .startObject("language")
                                                                     .startObject("en")
@@ -883,11 +797,11 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
                                                             .endObject()
                                                         .endObject();
 
-        assertXContentBuilder(expected, sampleWithIncludes("properties.**.name"));
+        testFilter(expected, SAMPLE, singleton("properties.**.name"), emptySet());
     }
 
     public void testRecurseField2Exclusive() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject()
+        final Builder expected = builder -> builder.startObject()
                                                             .field("title", "My awesome book")
                                                             .field("pages", 456)
                                                             .field("price", 27.99)
@@ -948,11 +862,11 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
                                                             .endObject()
                                                        .endObject();
 
-        assertXContentBuilder(expected, sampleWithExcludes("properties.**.name"));
+        testFilter(expected, SAMPLE, emptySet(), singleton("properties.**.name"));
     }
 
     public void testRecurseField3Inclusive() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject()
+        final Builder expected = builder -> builder.startObject()
                                                             .startObject("properties")
                                                                 .startObject("language")
                                                                     .startObject("en")
@@ -977,11 +891,11 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
                                                             .endObject()
                                                         .endObject();
 
-        assertXContentBuilder(expected, sampleWithIncludes("properties.*.en.**.name"));
+        testFilter(expected, SAMPLE, singleton("properties.*.en.**.name"), emptySet());
     }
 
     public void testRecurseField3Exclusive() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject()
+        final Builder expected = builder -> builder.startObject()
                                                             .field("title", "My awesome book")
                                                             .field("pages", 456)
                                                             .field("price", 27.99)
@@ -1047,11 +961,11 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
                                                             .endObject()
                                                        .endObject();
 
-        assertXContentBuilder(expected, sampleWithExcludes("properties.*.en.**.name"));
+        testFilter(expected, SAMPLE, emptySet(), singleton("properties.*.en.**.name"));
     }
 
     public void testRecurseField4Inclusive() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject()
+        final Builder expected = builder -> builder.startObject()
                                                             .startObject("properties")
                                                                 .startObject("language")
                                                                     .startObject("en")
@@ -1078,11 +992,11 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
                                                             .endObject()
                                                         .endObject();
 
-        assertXContentBuilder(expected, sampleWithIncludes("properties.**.distributors.name"));
+        testFilter(expected, SAMPLE, singleton("properties.**.distributors.name"), emptySet());
     }
 
     public void testRecurseField4Exclusive() throws Exception {
-        XContentBuilder expected = newXContentBuilder().startObject()
+        final Builder expected = builder -> builder.startObject()
                                                             .field("title", "My awesome book")
                                                             .field("pages", 456)
                                                             .field("price", 27.99)
@@ -1146,145 +1060,144 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
                                                             .endObject()
                                                        .endObject();
 
-        assertXContentBuilder(expected, sampleWithExcludes("properties.**.distributors.name"));
+        testFilter(expected, SAMPLE, emptySet(), singleton("properties.**.distributors.name"));
     }
 
     public void testRawField() throws Exception {
-        XContentBuilder expectedRawField = newXContentBuilder().startObject().field("foo", 0).startObject("raw")
-                .field("content", "hello world!").endObject().endObject();
-        XContentBuilder expectedRawFieldFiltered = newXContentBuilder().startObject().field("foo", 0).endObject();
-        XContentBuilder expectedRawFieldNotFiltered = newXContentBuilder().startObject().startObject("raw").field("content", "hello world!")
-                .endObject().endObject();
+        final Builder expectedRawField = builder -> builder
+                .startObject()
+                    .field("foo", 0)
+                    .startObject("raw")
+                        .field("content", "hello world!")
+                    .endObject()
+                .endObject();
+
+        final Builder expectedRawFieldFiltered = builder -> builder
+                .startObject()
+                    .field("foo", 0)
+                .endObject();
+
+        final Builder expectedRawFieldNotFiltered = builder -> builder
+                .startObject()
+                    .startObject("raw")
+                        .field("content", "hello world!")
+                    .endObject()
+                .endObject();
 
-        BytesReference raw = newXContentBuilder().startObject().field("content", "hello world!").endObject().bytes();
+        Builder sampleWithRaw = builder -> {
+            BytesReference raw = XContentBuilder.builder(builder.contentType().xContent())
+                    .startObject()
+                        .field("content", "hello world!")
+                    .endObject()
+                    .bytes();
+            return builder.startObject().field("foo", 0).rawField("raw", raw).endObject();
+        };
 
         // Test method: rawField(String fieldName, BytesReference content)
-        assertXContentBuilder(expectedRawField, newXContentBuilder().startObject().field("foo", 0).rawField("raw", raw).endObject());
-        assertXContentBuilder(expectedRawFieldFiltered,
-                newXContentBuilderWithIncludes("f*").startObject().field("foo", 0).rawField("raw", raw).endObject());
-        assertXContentBuilder(expectedRawFieldFiltered,
-                newXContentBuilderWithExcludes("r*").startObject().field("foo", 0).rawField("raw", raw).endObject());
-        assertXContentBuilder(expectedRawFieldNotFiltered,
-                newXContentBuilderWithIncludes("r*").startObject().field("foo", 0).rawField("raw", raw).endObject());
-        assertXContentBuilder(expectedRawFieldNotFiltered,
-                newXContentBuilderWithExcludes("f*").startObject().field("foo", 0).rawField("raw", raw).endObject());
+        testFilter(expectedRawField, sampleWithRaw, emptySet(), emptySet());
+        testFilter(expectedRawFieldFiltered, sampleWithRaw, singleton("f*"), emptySet());
+        testFilter(expectedRawFieldFiltered, sampleWithRaw, emptySet(), singleton("r*"));
+        testFilter(expectedRawFieldNotFiltered, sampleWithRaw, singleton("r*"), emptySet());
+        testFilter(expectedRawFieldNotFiltered, sampleWithRaw, emptySet(), singleton("f*"));
+
+        sampleWithRaw = builder -> {
+            BytesReference raw = XContentBuilder.builder(builder.contentType().xContent())
+                    .startObject()
+                    .   field("content", "hello world!")
+                    .endObject()
+                    .bytes();
+            return builder.startObject().field("foo", 0).rawField("raw", raw.streamInput()).endObject();
+        };
 
         // Test method: rawField(String fieldName, InputStream content)
-        assertXContentBuilder(expectedRawField,
-                newXContentBuilder().startObject().field("foo", 0).rawField("raw", raw.streamInput()).endObject());
-        assertXContentBuilder(expectedRawFieldFiltered, newXContentBuilderWithIncludes("f*").startObject().field("foo", 0)
-                .rawField("raw", raw.streamInput()).endObject());
-        assertXContentBuilder(expectedRawFieldFiltered, newXContentBuilderWithExcludes("r*").startObject().field("foo", 0)
-                .rawField("raw", raw.streamInput()).endObject());
-        assertXContentBuilder(expectedRawFieldNotFiltered, newXContentBuilderWithIncludes("r*").startObject().field("foo", 0)
-                .rawField("raw", raw.streamInput()).endObject());
-        assertXContentBuilder(expectedRawFieldNotFiltered, newXContentBuilderWithExcludes("f*").startObject().field("foo", 0)
-                .rawField("raw", raw.streamInput()).endObject());
+        testFilter(expectedRawField, sampleWithRaw, emptySet(), emptySet());
+        testFilter(expectedRawFieldFiltered, sampleWithRaw, singleton("f*"), emptySet());
+        testFilter(expectedRawFieldFiltered, sampleWithRaw, emptySet(), singleton("r*"));
+        testFilter(expectedRawFieldNotFiltered, sampleWithRaw, singleton("r*"), emptySet());
+        testFilter(expectedRawFieldNotFiltered, sampleWithRaw, emptySet(), singleton("f*"));
     }
 
     public void testArrays() throws Exception {
         // Test: Array of values (no filtering)
-        XContentBuilder expected = newXContentBuilder().startObject().startArray("tags").value("lorem").value("ipsum").value("dolor")
-                .endArray().endObject();
-        assertXContentBuilder(expected, newXContentBuilderWithIncludes("t*").startObject().startArray("tags").value("lorem").value("ipsum")
-                .value("dolor").endArray().endObject());
-        assertXContentBuilder(expected, newXContentBuilderWithIncludes("tags").startObject().startArray("tags").value("lorem")
-                .value("ipsum").value("dolor").endArray().endObject());
-        assertXContentBuilder(expected, newXContentBuilderWithExcludes("a").startObject().startArray("tags").value("lorem").value("ipsum")
-                .value("dolor").endArray().endObject());
+        final Builder sampleArrayOfValues = builder -> builder
+                .startObject()
+                    .startArray("tags")
+                        .value("lorem").value("ipsum").value("dolor")
+                    .endArray()
+                .endObject();
+        testFilter(sampleArrayOfValues, sampleArrayOfValues, singleton("t*"), emptySet());
+        testFilter(sampleArrayOfValues, sampleArrayOfValues, singleton("tags"), emptySet());
+        testFilter(sampleArrayOfValues, sampleArrayOfValues, emptySet(), singleton("a"));
 
         // Test: Array of values (with filtering)
-        assertXContentBuilder(newXContentBuilder().startObject().endObject(), newXContentBuilderWithIncludes("foo").startObject()
-                .startArray("tags").value("lorem").value("ipsum").value("dolor").endArray().endObject());
-        assertXContentBuilder(newXContentBuilder().startObject().endObject(), newXContentBuilderWithExcludes("t*").startObject()
-                .startArray("tags").value("lorem").value("ipsum").value("dolor").endArray().endObject());
-        assertXContentBuilder(newXContentBuilder().startObject().endObject(), newXContentBuilderWithExcludes("tags").startObject()
-                .startArray("tags").value("lorem").value("ipsum").value("dolor").endArray().endObject());
+        Builder expected = builder -> builder.startObject().endObject();
+        testFilter(expected, sampleArrayOfValues, singleton("foo"), emptySet());
+        testFilter(expected, sampleArrayOfValues, emptySet(), singleton("t*"));
+        testFilter(expected, sampleArrayOfValues, emptySet(), singleton("tags"));
 
         // Test: Array of objects (no filtering)
-        expected = newXContentBuilder().startObject().startArray("tags").startObject().field("lastname", "lorem").endObject().startObject()
-                .field("firstname", "ipsum").endObject().endArray().endObject();
-        assertXContentBuilder(expected, newXContentBuilderWithIncludes("t*").startObject().startArray("tags").startObject()
-                .field("lastname", "lorem").endObject().startObject().field("firstname", "ipsum").endObject().endArray().endObject());
-        assertXContentBuilder(expected, newXContentBuilderWithIncludes("tags").startObject().startArray("tags").startObject()
-                .field("lastname", "lorem").endObject().startObject().field("firstname", "ipsum").endObject().endArray().endObject());
-        assertXContentBuilder(expected, newXContentBuilderWithExcludes("a").startObject().startArray("tags").startObject()
-                .field("lastname", "lorem").endObject().startObject().field("firstname", "ipsum").endObject().endArray().endObject());
+        final Builder sampleArrayOfObjects = builder -> builder
+                .startObject()
+                    .startArray("tags")
+                        .startObject()
+                            .field("lastname", "lorem")
+                        .endObject()
+                        .startObject()
+                            .field("firstname", "ipsum")
+                        .endObject()
+                    .endArray()
+                .endObject();
+        testFilter(sampleArrayOfObjects, sampleArrayOfObjects, singleton("t*"), emptySet());
+        testFilter(sampleArrayOfObjects, sampleArrayOfObjects, singleton("tags"), emptySet());
+        testFilter(sampleArrayOfObjects, sampleArrayOfObjects, emptySet(), singleton("a"));
 
         // Test: Array of objects (with filtering)
-        assertXContentBuilder(newXContentBuilder().startObject().endObject(),
-                newXContentBuilderWithIncludes("foo").startObject().startArray("tags").startObject().field("lastname", "lorem").endObject()
-                        .startObject().field("firstname", "ipsum").endObject().endArray().endObject());
-        assertXContentBuilder(newXContentBuilder().startObject().endObject(),
-                newXContentBuilderWithExcludes("t*").startObject().startArray("tags").startObject().field("lastname", "lorem").endObject()
-                        .startObject().field("firstname", "ipsum").endObject().endArray().endObject());
-        assertXContentBuilder(newXContentBuilder().startObject().endObject(),
-                newXContentBuilderWithExcludes("tags").startObject().startArray("tags").startObject().field("lastname", "lorem").endObject()
-                        .startObject().field("firstname", "ipsum").endObject().endArray().endObject());
+        testFilter(expected, sampleArrayOfObjects, singleton("foo"), emptySet());
+        testFilter(expected, sampleArrayOfObjects, emptySet(), singleton("t*"));
+        testFilter(expected, sampleArrayOfObjects, emptySet(), singleton("tags"));
 
         // Test: Array of objects (with partial filtering)
-        expected = newXContentBuilder().startObject().startArray("tags").startObject().field("firstname", "ipsum").endObject().endArray()
+        expected = builder -> builder
+                .startObject()
+                    .startArray("tags")
+                        .startObject()
+                            .field("firstname", "ipsum")
+                        .endObject()
+                    .endArray()
                 .endObject();
-        assertXContentBuilder(expected, newXContentBuilderWithIncludes("t*.firstname").startObject().startArray("tags").startObject()
-                .field("lastname", "lorem").endObject().startObject().field("firstname", "ipsum").endObject().endArray().endObject());
-        assertXContentBuilder(expected, newXContentBuilderWithExcludes("t*.lastname").startObject().startArray("tags").startObject()
-                .field("lastname", "lorem").endObject().startObject().field("firstname", "ipsum").endObject().endArray().endObject());
+        testFilter(expected, sampleArrayOfObjects, singleton("t*.firstname"), emptySet());
+        testFilter(expected, sampleArrayOfObjects, emptySet(), singleton("t*.lastname"));
     }
 
     public void testEmptyObject() throws IOException {
-        final Function<XContentBuilder, XContentBuilder> build = builder -> {
-            try {
-                return builder.startObject().startObject("foo").endObject().endObject();
-            } catch (IOException e) {
-                throw new RuntimeException(e);
-            }
-        };
-
-        XContentBuilder expected = build.apply(newXContentBuilder());
-        assertXContentBuilder(expected, build.apply(newXContentBuilderWithIncludes("foo")));
-        assertXContentBuilder(expected, build.apply(newXContentBuilderWithExcludes("bar")));
-        assertXContentBuilder(expected, build.apply(newXContentBuilder(singleton("f*"), singleton("baz"))));
-
-        expected = newXContentBuilder().startObject().endObject();
-        assertXContentBuilder(expected, build.apply(newXContentBuilderWithExcludes("foo")));
-        assertXContentBuilder(expected, build.apply(newXContentBuilderWithIncludes("bar")));
-        assertXContentBuilder(expected, build.apply(newXContentBuilder(singleton("f*"), singleton("foo"))));
-    }
+        final Builder sample = builder -> builder.startObject().startObject("foo").endObject().endObject();
 
-    public void testSingleFieldObject() throws IOException {
-        final Function<XContentBuilder, XContentBuilder> build = builder -> {
-            try {
-                return builder.startObject().startObject("foo").field("bar", "test").endObject().endObject();
-            } catch (IOException e) {
-                throw new RuntimeException(e);
-            }
-        };
-
-        XContentBuilder expected = build.apply(newXContentBuilder());
-        assertXContentBuilder(expected, build.apply(newXContentBuilderWithIncludes("foo.bar")));
-        assertXContentBuilder(expected, build.apply(newXContentBuilderWithExcludes("foo.baz")));
-        assertXContentBuilder(expected, build.apply(newXContentBuilder(singleton("foo"), singleton("foo.baz"))));
+        Builder expected = builder -> builder.startObject().startObject("foo").endObject().endObject();
+        testFilter(expected, sample, singleton("foo"), emptySet());
+        testFilter(expected, sample, emptySet(), singleton("bar"));
+        testFilter(expected, sample, singleton("f*"), singleton("baz"));
 
-        expected = newXContentBuilder().startObject().endObject();
-        assertXContentBuilder(expected, build.apply(newXContentBuilderWithExcludes("foo.bar")));
-        assertXContentBuilder(expected, build.apply(newXContentBuilder(singleton("foo"), singleton("foo.b*"))));
+        expected = builder -> builder.startObject().endObject();
+        testFilter(expected, sample, emptySet(), singleton("foo"));
+        testFilter(expected, sample, singleton("bar"), emptySet());
+        testFilter(expected, sample, singleton("f*"), singleton("foo"));
     }
 
     public void testSingleFieldWithBothExcludesIncludes() throws IOException {
-        XContentBuilder expected = newXContentBuilder()
+        final Builder expected = builder -> builder
             .startObject()
                 .field("pages", 456)
                 .field("price", 27.99)
             .endObject();
 
-        assertXContentBuilder(expected, sampleWithFilters(singleton("p*"), singleton("properties")));
+        testFilter(expected, SAMPLE, singleton("p*"), singleton("properties"));
     }
 
     public void testObjectsInArrayWithBothExcludesIncludes() throws IOException {
         Set<String> includes = Sets.newHashSet("tags", "authors");
         Set<String> excludes = singleton("authors.name");
 
-        XContentBuilder expected = newXContentBuilder()
+        final Builder expected = builder -> builder
             .startObject()
                 .startArray("tags")
                     .value("elasticsearch")
@@ -1302,14 +1215,14 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
                 .endArray()
             .endObject();
 
-        assertXContentBuilder(expected, sampleWithFilters(includes, excludes));
+        testFilter(expected, SAMPLE, includes, excludes);
     }
 
     public void testRecursiveObjectsInArrayWithBothExcludesIncludes() throws IOException {
         Set<String> includes = Sets.newHashSet("**.language", "properties.weight");
         Set<String> excludes = singleton("**.distributors");
 
-        XContentBuilder expected = newXContentBuilder()
+        final Builder expected = builder -> builder
             .startObject()
                 .startObject("properties")
                     .field("weight", 0.8d)
@@ -1326,22 +1239,22 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
                 .endObject()
             .endObject();
 
-        assertXContentBuilder(expected, sampleWithFilters(includes, excludes));
+        testFilter(expected, SAMPLE, includes, excludes);
     }
 
     public void testRecursiveSameObjectWithBothExcludesIncludes() throws IOException {
         Set<String> includes = singleton("**.distributors");
         Set<String> excludes = singleton("**.distributors");
 
-        XContentBuilder expected = newXContentBuilder().startObject().endObject();
-        assertXContentBuilder(expected, sampleWithFilters(includes, excludes));
+        final Builder expected = builder -> builder.startObject().endObject();
+        testFilter(expected, SAMPLE, includes, excludes);
     }
 
     public void testRecursiveObjectsPropertiesWithBothExcludesIncludes() throws IOException {
         Set<String> includes = singleton("**.en.*");
         Set<String> excludes = Sets.newHashSet("**.distributors.*.name", "**.street");
 
-        XContentBuilder expected = newXContentBuilder()
+        final Builder expected = builder -> builder
             .startObject()
                 .startObject("properties")
                     .startObject("language")
@@ -1369,26 +1282,149 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
                 .endObject()
             .endObject();
 
-        assertXContentBuilder(expected, sampleWithFilters(includes, excludes));
+        testFilter(expected, SAMPLE, includes, excludes);
     }
 
     public void testWithLfAtEnd() throws IOException {
-        final Function<XContentBuilder, XContentBuilder> build = builder -> {
-            try {
-                return builder.startObject().startObject("foo").field("bar", "baz").endObject().endObject().prettyPrint().lfAtEnd();
-            } catch (IOException e) {
-                throw new RuntimeException(e);
-            }
-        };
+        final Builder sample = builder -> builder
+                .startObject()
+                    .startObject("foo")
+                        .field("bar", "baz")
+                    .endObject()
+                .endObject()
+                .prettyPrint()
+                .lfAtEnd();
+
+        testFilter(sample, sample, singleton("foo"), emptySet());
+        testFilter(sample, sample, emptySet(), singleton("bar"));
+        testFilter(sample, sample, singleton("f*"), singleton("baz"));
+
+        final Builder expected = builder -> builder.startObject().endObject().prettyPrint().lfAtEnd();
+        testFilter(expected, sample, emptySet(), singleton("foo"));
+        testFilter(expected, sample, singleton("bar"), emptySet());
+        testFilter(expected, sample, singleton("f*"), singleton("foo"));
+    }
+
+    public void testBasics() throws Exception {
+        final Builder sample = builder -> builder
+                .startObject()
+                    .field("test1", "value1")
+                    .field("test2", "value2")
+                    .field("something_else", "value3")
+                .endObject();
+
+        Builder expected = builder -> builder
+                .startObject()
+                    .field("test1", "value1")
+                .endObject();
+        testFilter(expected, sample, singleton("test1"), emptySet());
+
+        expected = builder -> builder
+                .startObject()
+                    .field("test1", "value1")
+                    .field("test2", "value2")
+                .endObject();
+        testFilter(expected, sample, singleton("test*"), emptySet());
+
+        expected = builder -> builder
+                .startObject()
+                    .field("test2", "value2")
+                    .field("something_else", "value3")
+                .endObject();
+        testFilter(expected, sample, emptySet(), singleton("test1"));
+
+        // more complex object...
+        final Builder complex = builder -> builder
+                .startObject()
+                    .startObject("path1")
+                        .startArray("path2")
+                            .startObject().field("test", "value1").endObject()
+                            .startObject().field("test", "value2").endObject()
+                        .endArray()
+                    .endObject()
+                    .field("test1", "value1")
+                .endObject();
+
+        expected = builder -> builder
+                .startObject()
+                    .startObject("path1")
+                        .startArray("path2")
+                            .startObject().field("test", "value1").endObject()
+                            .startObject().field("test", "value2").endObject()
+                        .endArray()
+                    .endObject()
+                .endObject();
+        testFilter(expected, complex, singleton("path1"), emptySet());
+        testFilter(expected, complex, singleton("path1*"), emptySet());
+        testFilter(expected, complex, singleton("path1.path2.*"), emptySet());
+
+        expected = builder -> builder
+                .startObject()
+                    .field("test1", "value1")
+                .endObject();
+        testFilter(expected, complex, singleton("test1*"), emptySet());
+    }
+
+    /**
+     * Generalization of {@link XContentMapValuesTests#testSupplementaryCharactersInPaths()}
+     */
+    public void testFilterSupplementaryCharactersInPaths() throws IOException {
+        final Builder sample = builder -> builder
+                .startObject()
+                    .field("搜索", 2)
+                    .field("指数", 3)
+                .endObject();
+
+        Builder expected = builder -> builder
+                .startObject()
+                    .field("搜索", 2)
+                .endObject();
+        testFilter(expected, sample, singleton("搜索"), emptySet());
 
-        XContentBuilder expected = build.apply(newXContentBuilder());
-        assertXContentBuilder(expected, build.apply(newXContentBuilderWithIncludes("foo")));
-        assertXContentBuilder(expected, build.apply(newXContentBuilderWithExcludes("bar")));
-        assertXContentBuilder(expected, build.apply(newXContentBuilder(singleton("f*"), singleton("baz"))));
+        expected = builder -> builder
+                .startObject()
+                    .field("指数", 3)
+                .endObject();
+        testFilter(expected, sample, emptySet(), singleton("搜索"));
+    }
 
-        expected = newXContentBuilder().startObject().endObject().prettyPrint().lfAtEnd();
-        assertXContentBuilder(expected, build.apply(newXContentBuilderWithExcludes("foo")));
-        assertXContentBuilder(expected, build.apply(newXContentBuilderWithIncludes("bar")));
-        assertXContentBuilder(expected, build.apply(newXContentBuilder(singleton("f*"), singleton("foo"))));
+    /**
+     * Generalization of {@link XContentMapValuesTests#testSharedPrefixes()}
+     */
+    public void testFilterSharedPrefixes() throws IOException {
+        final Builder sample = builder -> builder
+                .startObject()
+                    .field("foobar", 2)
+                    .field("foobaz", 3)
+                .endObject();
+
+        Builder expected = builder -> builder
+                .startObject()
+                    .field("foobar", 2)
+                .endObject();
+        testFilter(expected, sample, singleton("foobar"), emptySet());
+
+        expected = builder -> builder
+                .startObject()
+                    .field("foobaz", 3)
+                .endObject();
+        testFilter(expected, sample, emptySet(), singleton("foobar"));
+    }
+
+    /**
+     * Generalization of {@link XContentMapValuesTests#testPrefix()}
+     */
+    public void testFilterPrefix() throws IOException {
+        final Builder sample = builder -> builder
+                .startObject()
+                    .array("photos", "foo", "bar")
+                    .field("photosCount", 2)
+                .endObject();
+
+        Builder expected = builder -> builder
+                .startObject()
+                    .field("photosCount", 2)
+                .endObject();
+        testFilter(expected, sample, singleton("photosCount"), emptySet());
     }
 }

core/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java

@@ -21,14 +21,12 @@ package org.elasticsearch.common.xcontent.support;
 
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.test.ESTestCase;
-import org.hamcrest.Matchers;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -36,9 +34,10 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
-import static java.util.Collections.emptyMap;
-import static java.util.Collections.singletonMap;
+import static org.elasticsearch.common.xcontent.XContentHelper.convertToMap;
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
 import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.hasSize;
@@ -46,60 +45,29 @@ import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.core.IsEqual.equalTo;
 
-public class XContentMapValuesTests extends ESTestCase {
-    public void testFilter() throws Exception {
-        XContentBuilder builder = XContentFactory.jsonBuilder().startObject()
-                .field("test1", "value1")
-                .field("test2", "value2")
-                .field("something_else", "value3")
-                .endObject();
-
-        Map<String, Object> source;
-        try (XContentParser parser = createParser(JsonXContent.jsonXContent, builder.string())) {
-            source = parser.map();
-        }
-        Map<String, Object> filter = XContentMapValues.filter(source, new String[]{"test1"}, Strings.EMPTY_ARRAY);
-        assertThat(filter.size(), equalTo(1));
-        assertThat(filter.get("test1").toString(), equalTo("value1"));
-
-        filter = XContentMapValues.filter(source, new String[]{"test*"}, Strings.EMPTY_ARRAY);
-        assertThat(filter.size(), equalTo(2));
-        assertThat(filter.get("test1").toString(), equalTo("value1"));
-        assertThat(filter.get("test2").toString(), equalTo("value2"));
+public class XContentMapValuesTests extends AbstractFilteringTestCase {
 
-        filter = XContentMapValues.filter(source, Strings.EMPTY_ARRAY, new String[]{"test1"});
-        assertThat(filter.size(), equalTo(2));
-        assertThat(filter.get("test2").toString(), equalTo("value2"));
-        assertThat(filter.get("something_else").toString(), equalTo("value3"));
+    @Override
+    protected void testFilter(Builder expected, Builder actual, Set<String> includes, Set<String> excludes) throws IOException {
+        final XContentType xContentType = randomFrom(XContentType.values());
+        final boolean humanReadable = randomBoolean();
 
-        // more complex object...
-        builder = XContentFactory.jsonBuilder().startObject()
-                .startObject("path1")
-                .startArray("path2")
-                .startObject().field("test", "value1").endObject()
-                .startObject().field("test", "value2").endObject()
-                .endArray()
-                .endObject()
-                .field("test1", "value1")
-                .endObject();
-
-        try (XContentParser parser = createParser(JsonXContent.jsonXContent, builder.string())) {
-            source = parser.map();
+        String[] sourceIncludes;
+        if (includes == null) {
+            sourceIncludes = randomBoolean() ? Strings.EMPTY_ARRAY : null;
+        } else {
+            sourceIncludes = includes.toArray(new String[includes.size()]);
+        }
+        String[] sourceExcludes;
+        if (excludes == null) {
+            sourceExcludes = randomBoolean() ? Strings.EMPTY_ARRAY : null;
+        } else {
+            sourceExcludes = excludes.toArray(new String[excludes.size()]);
         }
-        filter = XContentMapValues.filter(source, new String[]{"path1"}, Strings.EMPTY_ARRAY);
-        assertThat(filter.size(), equalTo(1));
-
-        filter = XContentMapValues.filter(source, new String[]{"path1*"}, Strings.EMPTY_ARRAY);
-        assertThat(filter.get("path1"), equalTo(source.get("path1")));
-        assertThat(filter.containsKey("test1"), equalTo(false));
-
-        filter = XContentMapValues.filter(source, new String[]{"test1*"}, Strings.EMPTY_ARRAY);
-        assertThat(filter.get("test1"), equalTo(source.get("test1")));
-        assertThat(filter.containsKey("path1"), equalTo(false));
 
-        filter = XContentMapValues.filter(source, new String[]{"path1.path2.*"}, Strings.EMPTY_ARRAY);
-        assertThat(filter.get("path1"), equalTo(source.get("path1")));
-        assertThat(filter.containsKey("test1"), equalTo(false));
+        assertEquals("Filtered map must be equal to the expected map",
+                toMap(expected, xContentType, humanReadable),
+                XContentMapValues.filter(toMap(actual, xContentType, humanReadable), sourceIncludes, sourceExcludes));
     }
 
     @SuppressWarnings({"unchecked"})
@@ -376,7 +344,6 @@ public class XContentMapValuesTests extends ESTestCase {
         Map<String, Object> filteredMap = XContentMapValues.filter(map, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY);
         assertThat(filteredMap.size(), equalTo(1));
         assertThat(filteredMap.get("field").toString(), equalTo("value"));
-
     }
 
     public void testThatFilterIncludesEmptyObjectWhenUsingIncludes() throws Exception {
@@ -385,7 +352,7 @@ public class XContentMapValuesTests extends ESTestCase {
                 .endObject()
                 .endObject();
 
-        Tuple<XContentType, Map<String, Object>> mapTuple = XContentHelper.convertToMap(builder.bytes(), true, builder.contentType());
+        Tuple<XContentType, Map<String, Object>> mapTuple = convertToMap(builder.bytes(), true, builder.contentType());
         Map<String, Object> filteredSource = XContentMapValues.filter(mapTuple.v2(), new String[]{"obj"}, Strings.EMPTY_ARRAY);
 
         assertThat(mapTuple.v2(), equalTo(filteredSource));
@@ -397,7 +364,7 @@ public class XContentMapValuesTests extends ESTestCase {
                 .endObject()
                 .endObject();
 
-        Tuple<XContentType, Map<String, Object>> mapTuple = XContentHelper.convertToMap(builder.bytes(), true, builder.contentType());
+        Tuple<XContentType, Map<String, Object>> mapTuple = convertToMap(builder.bytes(), true, builder.contentType());
         Map<String, Object> filteredSource = XContentMapValues.filter(mapTuple.v2(), Strings.EMPTY_ARRAY, new String[]{"nonExistingField"});
 
         assertThat(mapTuple.v2(), equalTo(filteredSource));
@@ -410,7 +377,7 @@ public class XContentMapValuesTests extends ESTestCase {
                 .endObject()
                 .endObject();
 
-        Tuple<XContentType, Map<String, Object>> mapTuple = XContentHelper.convertToMap(builder.bytes(), true, builder.contentType());
+        Tuple<XContentType, Map<String, Object>> mapTuple = convertToMap(builder.bytes(), true, builder.contentType());
         Map<String, Object> filteredSource = XContentMapValues.filter(mapTuple.v2(), Strings.EMPTY_ARRAY, new String[]{"obj.f1"});
 
         assertThat(filteredSource.size(), equalTo(1));
@@ -430,7 +397,7 @@ public class XContentMapValuesTests extends ESTestCase {
                 .endObject();
 
         // implicit include
-        Tuple<XContentType, Map<String, Object>> mapTuple = XContentHelper.convertToMap(builder.bytes(), true, builder.contentType());
+        Tuple<XContentType, Map<String, Object>> mapTuple = convertToMap(builder.bytes(), true, builder.contentType());
         Map<String, Object> filteredSource = XContentMapValues.filter(mapTuple.v2(), Strings.EMPTY_ARRAY, new String[]{"*.obj2"});
 
         assertThat(filteredSource.size(), equalTo(1));
@@ -460,7 +427,7 @@ public class XContentMapValuesTests extends ESTestCase {
                 .endObject()
                 .endObject();
 
-        Tuple<XContentType, Map<String, Object>> mapTuple = XContentHelper.convertToMap(builder.bytes(), true, builder.contentType());
+        Tuple<XContentType, Map<String, Object>> mapTuple = convertToMap(builder.bytes(), true, builder.contentType());
         Map<String, Object> filteredSource = XContentMapValues.filter(mapTuple.v2(), new String[]{"*.obj2"}, Strings.EMPTY_ARRAY);
 
         assertThat(filteredSource.size(), equalTo(1));
@@ -470,85 +437,6 @@ public class XContentMapValuesTests extends ESTestCase {
         assertThat(((Map) ((Map) filteredSource.get("obj1")).get("obj2")).size(), equalTo(0));
     }
 
-    public void testEmptyList() throws IOException {
-        XContentBuilder builder = XContentFactory.jsonBuilder().startObject()
-                .startArray("some_array")
-                .endArray().endObject();
-
-        try (XContentParser parser = createParser(JsonXContent.jsonXContent, builder.string())) {
-            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
-            assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
-            assertEquals("some_array", parser.currentName());
-            if (random().nextBoolean()) {
-                // sometimes read the start array token, sometimes not
-                assertEquals(XContentParser.Token.START_ARRAY, parser.nextToken());
-            }
-            assertEquals(Collections.emptyList(), parser.list());
-        }
-    }
-
-    public void testSimpleList() throws IOException {
-        XContentBuilder builder = XContentFactory.jsonBuilder().startObject()
-                .startArray("some_array")
-                    .value(1)
-                    .value(3)
-                    .value(0)
-                .endArray().endObject();
-
-        try (XContentParser parser = createParser(JsonXContent.jsonXContent, builder.string())) {
-            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
-            assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
-            assertEquals("some_array", parser.currentName());
-            if (random().nextBoolean()) {
-                // sometimes read the start array token, sometimes not
-                assertEquals(XContentParser.Token.START_ARRAY, parser.nextToken());
-            }
-            assertEquals(Arrays.asList(1, 3, 0), parser.list());
-        }
-    }
-
-    public void testNestedList() throws IOException {
-        XContentBuilder builder = XContentFactory.jsonBuilder().startObject()
-                .startArray("some_array")
-                    .startArray().endArray()
-                    .startArray().value(1).value(3).endArray()
-                    .startArray().value(2).endArray()
-                .endArray().endObject();
-
-        try (XContentParser parser = createParser(JsonXContent.jsonXContent, builder.string())) {
-            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
-            assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
-            assertEquals("some_array", parser.currentName());
-            if (random().nextBoolean()) {
-                // sometimes read the start array token, sometimes not
-                assertEquals(XContentParser.Token.START_ARRAY, parser.nextToken());
-            }
-            assertEquals(
-                    Arrays.asList(Collections.<Integer>emptyList(), Arrays.asList(1, 3), Arrays.asList(2)),
-                    parser.list());
-        }
-    }
-
-    public void testNestedMapInList() throws IOException {
-        XContentBuilder builder = XContentFactory.jsonBuilder().startObject()
-                .startArray("some_array")
-                    .startObject().field("foo", "bar").endObject()
-                    .startObject().endObject()
-                .endArray().endObject();
-
-        try (XContentParser parser = createParser(JsonXContent.jsonXContent, builder.string())) {
-            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
-            assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
-            assertEquals("some_array", parser.currentName());
-            if (random().nextBoolean()) {
-                // sometimes read the start array token, sometimes not
-                assertEquals(XContentParser.Token.START_ARRAY, parser.nextToken());
-            }
-            assertEquals(
-                    Arrays.asList(singletonMap("foo", "bar"), emptyMap()),
-                    parser.list());
-        }
-    }
 
     public void testDotsInFieldNames() {
         Map<String, Object> map = new HashMap<>();
@@ -599,4 +487,9 @@ public class XContentMapValuesTests extends ESTestCase {
         expected.put("photosCount", 2);
         assertEquals(expected, filtered);
     }
+
+    private static Map<String, Object> toMap(Builder test, XContentType xContentType, boolean humanReadable) throws IOException {
+        ToXContentObject toXContent = (builder, params) -> test.apply(builder);
+        return convertToMap(toXContent(toXContent, xContentType, humanReadable), true, xContentType).v2();
+    }
 }

core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractXContentFilteringTestCase.java

@@ -0,0 +1,105 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.xcontent.support.filtering;
+
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.support.AbstractFilteringTestCase;
+
+import java.io.IOException;
+import java.util.Set;
+
+import static java.util.Collections.emptySet;
+import static java.util.Collections.singleton;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+
+public abstract class AbstractXContentFilteringTestCase extends AbstractFilteringTestCase {
+
+    protected final void testFilter(Builder expected, Builder actual, Set<String> includes, Set<String> excludes) throws IOException {
+        assertFilterResult(expected.apply(createBuilder()), actual.apply(createBuilder(includes, excludes)));
+    }
+
+    protected abstract void assertFilterResult(XContentBuilder expected, XContentBuilder actual);
+
+    protected abstract XContentType getXContentType();
+
+    private XContentBuilder createBuilder() throws IOException {
+        return XContentBuilder.builder(getXContentType().xContent());
+    }
+
+    private XContentBuilder createBuilder(Set<String> includes, Set<String> excludes) throws IOException {
+        return XContentBuilder.builder(getXContentType().xContent(), includes, excludes);
+    }
+
+    public void testSingleFieldObject() throws IOException {
+        final Builder sample = builder -> builder.startObject().startObject("foo").field("bar", "test").endObject().endObject();
+
+        Builder expected = builder -> builder.startObject().startObject("foo").field("bar", "test").endObject().endObject();
+        testFilter(expected, sample, singleton("foo.bar"), emptySet());
+        testFilter(expected, sample, emptySet(), singleton("foo.baz"));
+        testFilter(expected, sample, singleton("foo"), singleton("foo.baz"));
+
+        expected = builder -> builder.startObject().endObject();
+        testFilter(expected, sample, emptySet(), singleton("foo.bar"));
+        testFilter(expected, sample, singleton("foo"), singleton("foo.b*"));
+    }
+
+    static void assertXContentBuilderAsString(final XContentBuilder expected, final XContentBuilder actual) {
+        assertThat(actual.bytes().utf8ToString(), is(expected.bytes().utf8ToString()));
+    }
+
+    static void assertXContentBuilderAsBytes(final XContentBuilder expected, final XContentBuilder actual) {
+        try {
+            XContent xContent = XContentFactory.xContent(actual.contentType());
+            XContentParser jsonParser = xContent.createParser(NamedXContentRegistry.EMPTY, expected.bytes());
+            XContentParser testParser = xContent.createParser(NamedXContentRegistry.EMPTY, actual.bytes());
+
+            while (true) {
+                XContentParser.Token token1 = jsonParser.nextToken();
+                XContentParser.Token token2 = testParser.nextToken();
+                if (token1 == null) {
+                    assertThat(token2, nullValue());
+                    return;
+                }
+                assertThat(token1, equalTo(token2));
+                switch (token1) {
+                    case FIELD_NAME:
+                        assertThat(jsonParser.currentName(), equalTo(testParser.currentName()));
+                        break;
+                    case VALUE_STRING:
+                        assertThat(jsonParser.text(), equalTo(testParser.text()));
+                        break;
+                    case VALUE_NUMBER:
+                        assertThat(jsonParser.numberType(), equalTo(testParser.numberType()));
+                        assertThat(jsonParser.numberValue(), equalTo(testParser.numberValue()));
+                        break;
+                }
+            }
+        } catch (Exception e) {
+            fail("Fail to verify the result of the XContentBuilder: " + e.getMessage());
+        }
+    }
+}

core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/CborFilteringGeneratorTests.java → core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/CborXContentFilteringTests.java

@@ -22,7 +22,7 @@ package org.elasticsearch.common.xcontent.support.filtering;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
 
-public class CborFilteringGeneratorTests extends JsonFilteringGeneratorTests {
+public class CborXContentFilteringTests extends AbstractXContentFilteringTestCase {
 
     @Override
     protected XContentType getXContentType() {
@@ -30,7 +30,7 @@ public class CborFilteringGeneratorTests extends JsonFilteringGeneratorTests {
     }
 
     @Override
-    protected void assertXContentBuilder(XContentBuilder expected, XContentBuilder builder) {
-        assertBinary(expected, builder);
+    protected void assertFilterResult(XContentBuilder expected, XContentBuilder actual) {
+        assertXContentBuilderAsBytes(expected, actual);
     }
 }

core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/JsonFilteringGeneratorTests.java → core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/JsonXContentFilteringTests.java

@@ -22,7 +22,7 @@ package org.elasticsearch.common.xcontent.support.filtering;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
 
-public class JsonFilteringGeneratorTests extends AbstractFilteringJsonGeneratorTestCase {
+public class JsonXContentFilteringTests extends AbstractXContentFilteringTestCase {
 
     @Override
     protected XContentType getXContentType() {
@@ -30,7 +30,11 @@ public class JsonFilteringGeneratorTests extends AbstractFilteringJsonGeneratorT
     }
 
     @Override
-    protected void assertXContentBuilder(XContentBuilder expected, XContentBuilder builder) {
-        assertString(expected, builder);
+    protected void assertFilterResult(XContentBuilder expected, XContentBuilder actual) {
+        if (randomBoolean()) {
+            assertXContentBuilderAsString(expected, actual);
+        } else {
+            assertXContentBuilderAsBytes(expected, actual);
+        }
     }
 }

core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/SmileFilteringGeneratorTests.java

@@ -22,7 +22,7 @@ package org.elasticsearch.common.xcontent.support.filtering;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
 
-public class SmileFilteringGeneratorTests extends JsonFilteringGeneratorTests {
+public class SmileFilteringGeneratorTests extends AbstractXContentFilteringTestCase {
 
     @Override
     protected XContentType getXContentType() {
@@ -30,7 +30,7 @@ public class SmileFilteringGeneratorTests extends JsonFilteringGeneratorTests {
     }
 
     @Override
-    protected void assertXContentBuilder(XContentBuilder expected, XContentBuilder builder) {
-        assertBinary(expected, builder);
+    protected void assertFilterResult(XContentBuilder expected, XContentBuilder actual) {
+        assertXContentBuilderAsBytes(expected, actual);
     }
 }

core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/YamlFilteringGeneratorTests.java

@@ -22,7 +22,7 @@ package org.elasticsearch.common.xcontent.support.filtering;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
 
-public class YamlFilteringGeneratorTests extends AbstractFilteringJsonGeneratorTestCase {
+public class YamlFilteringGeneratorTests extends AbstractXContentFilteringTestCase {
 
     @Override
     protected XContentType getXContentType() {
@@ -30,7 +30,11 @@ public class YamlFilteringGeneratorTests extends AbstractFilteringJsonGeneratorT
     }
 
     @Override
-    protected void assertXContentBuilder(XContentBuilder expected, XContentBuilder builder) {
-        assertString(expected, builder);
+    protected void assertFilterResult(XContentBuilder expected, XContentBuilder actual) {
+        if (randomBoolean()) {
+            assertXContentBuilderAsString(expected, actual);
+        } else {
+            assertXContentBuilderAsBytes(expected, actual);
+        }
     }
 }