add support for includes and excludes in MultiValuesSourceFieldConfig (#92396)

add support for includes and excludes in MultiValuesSourceFieldConfig, this will allow
aggregations based on it implement includes and excludes based on regular
expressions or a list of values.

server/src/main/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregationBuilder.java

@@ -46,8 +46,8 @@ public class WeightedAvgAggregationBuilder extends MultiValuesSourceAggregationB
     );
     static {
         MultiValuesSourceParseHelper.declareCommon(PARSER, true, ValueType.NUMERIC);
-        MultiValuesSourceParseHelper.declareField(VALUE_FIELD.getPreferredName(), PARSER, true, false, false, false);
-        MultiValuesSourceParseHelper.declareField(WEIGHT_FIELD.getPreferredName(), PARSER, true, false, false, false);
+        MultiValuesSourceParseHelper.declareField(VALUE_FIELD.getPreferredName(), PARSER, true, false, false, false, false);
+        MultiValuesSourceParseHelper.declareField(WEIGHT_FIELD.getPreferredName(), PARSER, true, false, false, false, false);
     }
 
     public static void registerUsage(ValuesSourceRegistry.Builder builder) {

server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceFieldConfig.java

@@ -17,6 +17,7 @@ import org.elasticsearch.index.query.AbstractQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude;
 import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContentObject;
@@ -38,6 +39,7 @@ public class MultiValuesSourceFieldConfig implements Writeable, ToXContentObject
     private final QueryBuilder filter;
     // supported only if heterogeneous == true
     private final ValueType userValueTypeHint;
+    private final IncludeExclude includeExclude;
     private final String format;
 
     private static final String NAME = "field_config";
@@ -50,6 +52,7 @@ public class MultiValuesSourceFieldConfig implements Writeable, ToXContentObject
      * @param timezoneAware - allows specifying timezone
      * @param filtered - allows specifying filters on the values
      * @param heterogeneous - allows specifying value-source specific format and user value type hint
+     * @param supportsIncludesExcludes - allows specifying includes and excludes
      * @param <C> - parser context
      * @return configured parser
      */
@@ -57,7 +60,8 @@ public class MultiValuesSourceFieldConfig implements Writeable, ToXContentObject
         boolean scriptable,
         boolean timezoneAware,
         boolean filtered,
-        boolean heterogeneous
+        boolean heterogeneous,
+        boolean supportsIncludesExcludes
     ) {
 
         ObjectParser<MultiValuesSourceFieldConfig.Builder, C> parser = new ObjectParser<>(
@@ -116,6 +120,23 @@ public class MultiValuesSourceFieldConfig implements Writeable, ToXContentObject
                 ObjectParser.ValueType.STRING
             );
         }
+
+        if (supportsIncludesExcludes) {
+            parser.declareField(
+                (b, v) -> b.setIncludeExclude(IncludeExclude.merge(v, b.getIncludeExclude())),
+                IncludeExclude::parseInclude,
+                IncludeExclude.INCLUDE_FIELD,
+                ObjectParser.ValueType.OBJECT_ARRAY_OR_STRING
+            );
+
+            parser.declareField(
+                (b, v) -> b.setIncludeExclude(IncludeExclude.merge(b.getIncludeExclude(), v)),
+                IncludeExclude::parseExclude,
+                IncludeExclude.EXCLUDE_FIELD,
+                ObjectParser.ValueType.STRING_ARRAY
+            );
+        }
+
         return parser;
     };
 
@@ -126,7 +147,8 @@ public class MultiValuesSourceFieldConfig implements Writeable, ToXContentObject
         ZoneId timeZone,
         QueryBuilder filter,
         ValueType userValueTypeHint,
-        String format
+        String format,
+        IncludeExclude includeExclude
     ) {
         this.fieldName = fieldName;
         this.missing = missing;
@@ -135,6 +157,7 @@ public class MultiValuesSourceFieldConfig implements Writeable, ToXContentObject
         this.filter = filter;
         this.userValueTypeHint = userValueTypeHint;
         this.format = format;
+        this.includeExclude = includeExclude;
     }
 
     public MultiValuesSourceFieldConfig(StreamInput in) throws IOException {
@@ -158,6 +181,11 @@ public class MultiValuesSourceFieldConfig implements Writeable, ToXContentObject
             this.userValueTypeHint = null;
             this.format = null;
         }
+        if (in.getVersion().onOrAfter(Version.V_8_7_0)) {
+            this.includeExclude = in.readOptionalWriteable(IncludeExclude::new);
+        } else {
+            this.includeExclude = null;
+        }
     }
 
     public Object getMissing() {
@@ -188,6 +216,10 @@ public class MultiValuesSourceFieldConfig implements Writeable, ToXContentObject
         return format;
     }
 
+    public IncludeExclude getIncludeExclude() {
+        return includeExclude;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         if (out.getVersion().onOrAfter(Version.V_7_6_0)) {
@@ -205,6 +237,9 @@ public class MultiValuesSourceFieldConfig implements Writeable, ToXContentObject
             out.writeOptionalWriteable(userValueTypeHint);
             out.writeOptionalString(format);
         }
+        if (out.getVersion().onOrAfter(Version.V_8_7_0)) {
+            out.writeOptionalWriteable(includeExclude);
+        }
     }
 
     @Override
@@ -232,6 +267,10 @@ public class MultiValuesSourceFieldConfig implements Writeable, ToXContentObject
         if (format != null) {
             builder.field(AggregationBuilder.CommonFields.FORMAT.getPreferredName(), format);
         }
+        if (includeExclude != null) {
+            includeExclude.toXContent(builder, params);
+        }
+
         builder.endObject();
         return builder;
     }
@@ -247,12 +286,13 @@ public class MultiValuesSourceFieldConfig implements Writeable, ToXContentObject
             && Objects.equals(timeZone, that.timeZone)
             && Objects.equals(filter, that.filter)
             && Objects.equals(userValueTypeHint, that.userValueTypeHint)
-            && Objects.equals(format, that.format);
+            && Objects.equals(format, that.format)
+            && Objects.equals(includeExclude, that.includeExclude);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(fieldName, missing, script, timeZone, filter, userValueTypeHint, format);
+        return Objects.hash(fieldName, missing, script, timeZone, filter, userValueTypeHint, format, includeExclude);
     }
 
     @Override
@@ -268,6 +308,7 @@ public class MultiValuesSourceFieldConfig implements Writeable, ToXContentObject
         private QueryBuilder filter = null;
         private ValueType userValueTypeHint = null;
         private String format = null;
+        private IncludeExclude includeExclude = null;
 
         public String getFieldName() {
             return fieldName;
@@ -328,6 +369,15 @@ public class MultiValuesSourceFieldConfig implements Writeable, ToXContentObject
             return format;
         }
 
+        public Builder setIncludeExclude(IncludeExclude includeExclude) {
+            this.includeExclude = includeExclude;
+            return this;
+        }
+
+        public IncludeExclude getIncludeExclude() {
+            return includeExclude;
+        }
+
         public MultiValuesSourceFieldConfig build() {
             if (Strings.isNullOrEmpty(fieldName) && script == null) {
                 throw new IllegalArgumentException(
@@ -351,7 +401,16 @@ public class MultiValuesSourceFieldConfig implements Writeable, ToXContentObject
                 );
             }
 
-            return new MultiValuesSourceFieldConfig(fieldName, missing, script, timeZone, filter, userValueTypeHint, format);
+            return new MultiValuesSourceFieldConfig(
+                fieldName,
+                missing,
+                script,
+                timeZone,
+                filter,
+                userValueTypeHint,
+                format,
+                includeExclude
+            );
         }
     }
 }

server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceParseHelper.java

@@ -65,12 +65,19 @@ public final class MultiValuesSourceParseHelper {
         boolean scriptable,
         boolean timezoneAware,
         boolean filterable,
-        boolean heterogeneous
+        boolean heterogeneous,
+        boolean supportsIncludesExcludes
     ) {
 
         objectParser.declareField(
             (o, fieldConfig) -> o.field(fieldName, fieldConfig.build()),
-            (p, c) -> MultiValuesSourceFieldConfig.parserBuilder(scriptable, timezoneAware, filterable, heterogeneous).parse(p, null),
+            (p, c) -> MultiValuesSourceFieldConfig.parserBuilder(
+                scriptable,
+                timezoneAware,
+                filterable,
+                heterogeneous,
+                supportsIncludesExcludes
+            ).parse(p, null),
             new ParseField(fieldName),
             ObjectParser.ValueType.OBJECT
         );

server/src/test/java/org/elasticsearch/search/aggregations/support/IncludeExcludeTests.java

@@ -35,6 +35,35 @@ import java.util.TreeSet;
 import static org.hamcrest.Matchers.equalTo;
 
 public class IncludeExcludeTests extends ESTestCase {
+
+    public static IncludeExclude randomIncludeExclude() {
+        switch (randomInt(7)) {
+            case 0:
+                return new IncludeExclude("incl*de", null, null, null);
+            case 1:
+                return new IncludeExclude("incl*de", "excl*de", null, null);
+            case 2:
+                return new IncludeExclude("incl*de", null, null, new TreeSet<>(Set.of(newBytesRef("exclude"))));
+            case 3:
+                return new IncludeExclude(null, "excl*de", null, null);
+            case 4:
+                return new IncludeExclude(null, "excl*de", new TreeSet<>(Set.of(newBytesRef("include"))), null);
+            case 5:
+                return new IncludeExclude(null, null, new TreeSet<>(Set.of(newBytesRef("include"))), null);
+            case 6:
+                return new IncludeExclude(
+                    null,
+                    null,
+                    new TreeSet<>(Set.of(newBytesRef("include"))),
+                    new TreeSet<>(Set.of(newBytesRef("exclude")))
+                );
+            case 7:
+                return new IncludeExclude(null, null, null, new TreeSet<>(Set.of(newBytesRef("exclude"))));
+            default:
+                throw new IllegalArgumentException("got unexpected parameter, expected 0 <= x <= 7");
+        }
+    }
+
     public void testEmptyTermsWithOrds() throws IOException {
         IncludeExclude inexcl = new IncludeExclude(null, null, new TreeSet<>(Set.of(new BytesRef("foo"))), null);
         OrdinalsFilter filter = inexcl.convertToOrdinalsFilter(DocValueFormat.RAW);

server/src/test/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceFieldConfigTests.java

@@ -15,6 +15,7 @@ import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude;
 import org.elasticsearch.test.AbstractXContentSerializingTestCase;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.XContentParser;
@@ -30,7 +31,7 @@ public class MultiValuesSourceFieldConfigTests extends AbstractXContentSerializi
 
     @Override
     protected MultiValuesSourceFieldConfig doParseInstance(XContentParser parser) throws IOException {
-        return MultiValuesSourceFieldConfig.parserBuilder(true, true, true, true).apply(parser, null).build();
+        return MultiValuesSourceFieldConfig.parserBuilder(true, true, true, true, true).apply(parser, null).build();
     }
 
     @Override
@@ -43,6 +44,7 @@ public class MultiValuesSourceFieldConfigTests extends AbstractXContentSerializi
         ValueType userValueTypeHint = randomBoolean()
             ? randomFrom(ValueType.STRING, ValueType.DOUBLE, ValueType.LONG, ValueType.DATE, ValueType.IP, ValueType.BOOLEAN)
             : null;
+        IncludeExclude includeExclude = randomBoolean() ? IncludeExcludeTests.randomIncludeExclude() : null;
         return new MultiValuesSourceFieldConfig.Builder().setFieldName(field)
             .setMissing(missing)
             .setScript(null)
@@ -50,6 +52,7 @@ public class MultiValuesSourceFieldConfigTests extends AbstractXContentSerializi
             .setFilter(filter)
             .setFormat(format)
             .setUserValueTypeHint(userValueTypeHint)
+            .setIncludeExclude(includeExclude)
             .build();
     }
 

x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/multiterms/MultiTermsAggregationBuilder.java

@@ -64,7 +64,8 @@ public class MultiTermsAggregationBuilder extends AbstractAggregationBuilder<Mul
             true,
             true,
             false,
-            true
+            true,
+            false
         );
 
         PARSER.declareBoolean(MultiTermsAggregationBuilder::showTermDocCountError, MultiTermsAggregationBuilder.SHOW_TERM_DOC_COUNT_ERROR);

x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregationBuilder.java

@@ -96,6 +96,7 @@ public class TopMetricsAggregationBuilder extends AbstractAggregationBuilder<Top
             true,
             false,
             false,
+            false,
             false
         );
         PARSER.declareObjectArray(constructorArg(), (p, n) -> metricParser.parse(p, null).build(), METRIC_FIELD);

x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregationBuilder.java

@@ -46,8 +46,8 @@ public class TTestAggregationBuilder extends MultiValuesSourceAggregationBuilder
 
     static {
         MultiValuesSourceParseHelper.declareCommon(PARSER, true, ValueType.NUMERIC);
-        MultiValuesSourceParseHelper.declareField(A_FIELD.getPreferredName(), PARSER, true, false, true, false);
-        MultiValuesSourceParseHelper.declareField(B_FIELD.getPreferredName(), PARSER, true, false, true, false);
+        MultiValuesSourceParseHelper.declareField(A_FIELD.getPreferredName(), PARSER, true, false, true, false, false);
+        MultiValuesSourceParseHelper.declareField(B_FIELD.getPreferredName(), PARSER, true, false, true, false, false);
         PARSER.declareString(TTestAggregationBuilder::testType, TYPE_FIELD);
         PARSER.declareInt(TTestAggregationBuilder::tails, TAILS_FIELD);
     }

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetsAggregationBuilder.java

@@ -64,7 +64,8 @@ public final class FrequentItemSetsAggregationBuilder extends AbstractAggregatio
             false,  // scriptable
             false,  // timezone aware
             false,  // filtered (not defined per field, but for all fields below)
-            false   // format
+            false,  // format
+            false   // includes and excludes
         );
         PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), (p, n) -> fieldsParser.parse(p, null).build(), FIELDS);
         PARSER.declareDouble(ConstructingObjectParser.optionalConstructorArg(), MINIMUM_SUPPORT);

x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregationBuilder.java

@@ -50,8 +50,8 @@ public class GeoLineAggregationBuilder extends MultiValuesSourceAggregationBuild
     );
     static {
         MultiValuesSourceParseHelper.declareCommon(PARSER, true, ValueType.NUMERIC);
-        MultiValuesSourceParseHelper.declareField(POINT_FIELD.getPreferredName(), PARSER, true, false, false, false);
-        MultiValuesSourceParseHelper.declareField(SORT_FIELD.getPreferredName(), PARSER, true, false, false, false);
+        MultiValuesSourceParseHelper.declareField(POINT_FIELD.getPreferredName(), PARSER, true, false, false, false, false);
+        MultiValuesSourceParseHelper.declareField(SORT_FIELD.getPreferredName(), PARSER, true, false, false, false, false);
         PARSER.declareString((builder, order) -> builder.sortOrder(SortOrder.fromString(order)), ORDER_FIELD);
         PARSER.declareBoolean(GeoLineAggregationBuilder::includeSort, INCLUDE_SORT_FIELD);
         PARSER.declareInt(GeoLineAggregationBuilder::size, SIZE_FIELD);