New AdjacencyMatrix aggregation

Similar to the Filters aggregation but only supports "keyed" filter buckets and automatically "ANDs" pairs of filters to produce a form of adjacency matrix.
The intersection of buckets "A" and "B" is named "A&B" (the choice of separator is configurable). Empty intersection buckets are removed from the final results.

Closes #22169

core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java

@@ -108,6 +108,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
         IndexSettings.INDEX_REFRESH_INTERVAL_SETTING,
         IndexSettings.MAX_RESULT_WINDOW_SETTING,
         IndexSettings.MAX_RESCORE_WINDOW_SETTING,
+        IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING,
         IndexSettings.INDEX_TRANSLOG_SYNC_INTERVAL_SETTING,
         IndexSettings.DEFAULT_FIELD_SETTING,
         IndexSettings.QUERY_STRING_LENIENT_SETTING,

core/src/main/java/org/elasticsearch/index/IndexSettings.java

@@ -97,6 +97,13 @@ public final class IndexSettings {
      */
     public static final Setting<Integer> MAX_RESCORE_WINDOW_SETTING =
             Setting.intSetting("index.max_rescore_window", MAX_RESULT_WINDOW_SETTING, 1, Property.Dynamic, Property.IndexScope);
+    /**
+     * Index setting describing the maximum number of filters clauses that can be used
+     * in an adjacency_matrix aggregation. The max number of buckets produced by  
+     * N filters is (N*N)/2 so a limit of 100 filters is imposed by default.
+     */
+    public static final Setting<Integer> MAX_ADJACENCY_MATRIX_FILTERS_SETTING =
+        Setting.intSetting("index.max_adjacency_matrix_filters", 100, 2, Property.Dynamic, Property.IndexScope);    
     public static final TimeValue DEFAULT_REFRESH_INTERVAL = new TimeValue(1, TimeUnit.SECONDS);
     public static final Setting<TimeValue> INDEX_REFRESH_INTERVAL_SETTING =
         Setting.timeSetting("index.refresh_interval", DEFAULT_REFRESH_INTERVAL, new TimeValue(-1, TimeUnit.MILLISECONDS),
@@ -155,6 +162,7 @@ public final class IndexSettings {
     private long gcDeletesInMillis = DEFAULT_GC_DELETES.millis();
     private volatile boolean warmerEnabled;
     private volatile int maxResultWindow;
+    private volatile int maxAdjacencyMatrixFilters;
     private volatile int maxRescoreWindow;
     private volatile boolean TTLPurgeDisabled;
     /**
@@ -246,6 +254,7 @@ public final class IndexSettings {
         gcDeletesInMillis = scopedSettings.get(INDEX_GC_DELETES_SETTING).getMillis();
         warmerEnabled = scopedSettings.get(INDEX_WARMER_ENABLED_SETTING);
         maxResultWindow = scopedSettings.get(MAX_RESULT_WINDOW_SETTING);
+        maxAdjacencyMatrixFilters = scopedSettings.get(MAX_ADJACENCY_MATRIX_FILTERS_SETTING);
         maxRescoreWindow = scopedSettings.get(MAX_RESCORE_WINDOW_SETTING);
         TTLPurgeDisabled = scopedSettings.get(INDEX_TTL_DISABLE_PURGE_SETTING);
         maxRefreshListeners = scopedSettings.get(MAX_REFRESH_LISTENERS_PER_SHARD);
@@ -267,6 +276,7 @@ public final class IndexSettings {
         scopedSettings.addSettingsUpdateConsumer(INDEX_TRANSLOG_DURABILITY_SETTING, this::setTranslogDurability);
         scopedSettings.addSettingsUpdateConsumer(INDEX_TTL_DISABLE_PURGE_SETTING, this::setTTLPurgeDisabled);
         scopedSettings.addSettingsUpdateConsumer(MAX_RESULT_WINDOW_SETTING, this::setMaxResultWindow);
+        scopedSettings.addSettingsUpdateConsumer(MAX_ADJACENCY_MATRIX_FILTERS_SETTING, this::setMaxAdjacencyMatrixFilters);
         scopedSettings.addSettingsUpdateConsumer(MAX_RESCORE_WINDOW_SETTING, this::setMaxRescoreWindow);
         scopedSettings.addSettingsUpdateConsumer(INDEX_WARMER_ENABLED_SETTING, this::setEnableWarmer);
         scopedSettings.addSettingsUpdateConsumer(INDEX_GC_DELETES_SETTING, this::setGCDeletes);
@@ -466,6 +476,17 @@ public final class IndexSettings {
     private void setMaxResultWindow(int maxResultWindow) {
         this.maxResultWindow = maxResultWindow;
     }
+    
+    /**
+     * Returns the max number of filters in adjacency_matrix aggregation search requests
+     */
+    public int getMaxAdjacencyMatrixFilters() {
+        return this.maxAdjacencyMatrixFilters;
+    }
+
+    private void setMaxAdjacencyMatrixFilters(int maxAdjacencyFilters) {
+        this.maxAdjacencyMatrixFilters = maxAdjacencyFilters;
+    }    
 
     /**
      * Returns the maximum rescore window for search requests.

core/src/main/java/org/elasticsearch/search/SearchModule.java

@@ -101,6 +101,8 @@ import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.BaseAggregationBuilder;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.adjacency.AdjacencyMatrixAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.adjacency.InternalAdjacencyMatrix;
 import org.elasticsearch.search.aggregations.bucket.children.ChildrenAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.children.InternalChildren;
 import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
@@ -373,6 +375,8 @@ public class SearchModule {
                 FilterAggregationBuilder::parse).addResultReader(InternalFilter::new));
         registerAggregation(new AggregationSpec(FiltersAggregationBuilder.NAME, FiltersAggregationBuilder::new,
                 FiltersAggregationBuilder::parse).addResultReader(InternalFilters::new));
+        registerAggregation(new AggregationSpec(AdjacencyMatrixAggregationBuilder.NAME, AdjacencyMatrixAggregationBuilder::new,
+                AdjacencyMatrixAggregationBuilder.getParser()).addResultReader(InternalAdjacencyMatrix::new));
         registerAggregation(new AggregationSpec(SamplerAggregationBuilder.NAME, SamplerAggregationBuilder::new,
                 SamplerAggregationBuilder::parse)
                     .addResultReader(InternalSampler.NAME, InternalSampler::new)

core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java

@@ -21,6 +21,8 @@ package org.elasticsearch.search.aggregations;
 import org.elasticsearch.common.geo.GeoDistance;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.search.aggregations.bucket.adjacency.AdjacencyMatrix;
+import org.elasticsearch.search.aggregations.bucket.adjacency.AdjacencyMatrixAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.children.Children;
 import org.elasticsearch.search.aggregations.bucket.children.ChildrenAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.filter.Filter;
@@ -82,6 +84,8 @@ import org.elasticsearch.search.aggregations.metrics.tophits.TopHitsAggregationB
 import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCount;
 import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCountAggregationBuilder;
 
+import java.util.Map;
+
 /**
  * Utility class to create aggregations.
  */
@@ -159,6 +163,20 @@ public class AggregationBuilders {
     public static FiltersAggregationBuilder filters(String name, QueryBuilder... filters) {
         return new FiltersAggregationBuilder(name, filters);
     }
+    
+    /**
+     * Create a new {@link AdjacencyMatrix} aggregation with the given name.
+     */
+    public static AdjacencyMatrixAggregationBuilder adjacencyMatrix(String name, Map<String, QueryBuilder> filters) {
+        return new AdjacencyMatrixAggregationBuilder(name, filters);
+    }    
+    
+    /**
+     * Create a new {@link AdjacencyMatrix} aggregation with the given name and separator
+     */
+    public static AdjacencyMatrixAggregationBuilder adjacencyMatrix(String name, String separator,  Map<String, QueryBuilder> filters) {
+        return new AdjacencyMatrixAggregationBuilder(name, separator, filters);
+    }     
 
     /**
      * Create a new {@link Sampler} aggregation with the given name.

core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrix.java

@@ -0,0 +1,48 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.adjacency;
+
+import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
+
+import java.util.List;
+
+/**
+ * A multi bucket aggregation where the buckets are defined by a set of filters
+ * (a bucket is produced per filter plus a bucket for each non-empty filter 
+ * intersection so A, B and A&B).
+ */
+public interface AdjacencyMatrix extends MultiBucketsAggregation {
+
+    /**
+     * A bucket associated with a specific filter or pair (identified by its
+     * key)
+     */
+    interface Bucket extends MultiBucketsAggregation.Bucket {
+    }
+
+    /**
+     * The buckets created by this aggregation.
+     */
+    @Override
+    List<? extends Bucket> getBuckets();
+
+    Bucket getBucketByKey(String key);
+
+}

core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregationBuilder.java

@@ -0,0 +1,237 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.adjacency;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
+import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.bucket.adjacency.AdjacencyMatrixAggregator.KeyedFilter;
+import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.query.QueryPhaseExecutionException;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+
+public class AdjacencyMatrixAggregationBuilder extends AbstractAggregationBuilder<AdjacencyMatrixAggregationBuilder> {
+    public static final String NAME = "adjacency_matrix";
+
+    private static final String DEFAULT_SEPARATOR = "&";
+
+    private static final ParseField SEPARATOR_FIELD = new ParseField("separator");
+    private static final ParseField FILTERS_FIELD = new ParseField("filters");
+    private List<KeyedFilter> filters;
+    private String separator = DEFAULT_SEPARATOR;
+
+    public static Aggregator.Parser getParser() {
+        ObjectParser<AdjacencyMatrixAggregationBuilder, QueryParseContext> parser = new ObjectParser<>(
+                AdjacencyMatrixAggregationBuilder.NAME);
+        parser.declareString(AdjacencyMatrixAggregationBuilder::separator, SEPARATOR_FIELD);
+        parser.declareNamedObjects(AdjacencyMatrixAggregationBuilder::setFiltersAsList, KeyedFilter.PARSER, FILTERS_FIELD);
+        return new Aggregator.Parser() {
+            @Override
+            public AggregationBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
+                AdjacencyMatrixAggregationBuilder result = parser.parse(context.parser(),
+                        new AdjacencyMatrixAggregationBuilder(aggregationName), context);
+                result.checkConsistency();
+                return result;
+            }
+        };
+    }
+
+    protected void checkConsistency() {
+        if ((filters == null) || (filters.size() == 0)) {
+            throw new IllegalStateException("[" + name  + "] is missing : " + FILTERS_FIELD.getPreferredName() + " parameter");
+        }        
+    }
+
+
+    protected void setFiltersAsMap(Map<String, QueryBuilder> filters) {
+        // Convert uniquely named objects into internal KeyedFilters
+        this.filters = new ArrayList<>(filters.size());
+        for (Entry<String, QueryBuilder> kv : filters.entrySet()) {
+            this.filters.add(new KeyedFilter(kv.getKey(), kv.getValue()));
+        }
+        // internally we want to have a fixed order of filters, regardless of
+        // the order of the filters in the request
+        Collections.sort(this.filters, Comparator.comparing(KeyedFilter::key));
+    }
+
+    protected void setFiltersAsList(List<KeyedFilter> filters) {
+        this.filters = new ArrayList<>(filters);
+        // internally we want to have a fixed order of filters, regardless of
+        // the order of the filters in the request
+        Collections.sort(this.filters, Comparator.comparing(KeyedFilter::key));
+    }
+    
+   
+    /**
+     * @param name
+     *            the name of this aggregation
+     */
+    protected AdjacencyMatrixAggregationBuilder(String name) {
+        super(name);
+    }    
+    
+    
+    /**
+     * @param name
+     *            the name of this aggregation
+     * @param filters
+     *            the filters and their keys to use with this aggregation.
+     */
+    public AdjacencyMatrixAggregationBuilder(String name, Map<String, QueryBuilder> filters) {
+        this(name, DEFAULT_SEPARATOR, filters);
+    }
+
+    /**
+     * @param name
+     *            the name of this aggregation
+     * @param separator
+     *            the string used to separate keys in intersections buckets e.g.
+     *            &amp; character for keyed filters A and B would return an
+     *            intersection bucket named A&amp;B
+     * @param filters
+     *            the filters and their key to use with this aggregation.
+     */
+    public AdjacencyMatrixAggregationBuilder(String name, String separator, Map<String, QueryBuilder> filters) {
+        super(name);
+        this.separator = separator;
+        setFiltersAsMap(filters);
+    }
+
+    /**
+     * Read from a stream.
+     */
+    public AdjacencyMatrixAggregationBuilder(StreamInput in) throws IOException {
+        super(in);
+        int filtersSize = in.readVInt();
+        separator = in.readString();
+        filters = new ArrayList<>(filtersSize);
+        for (int i = 0; i < filtersSize; i++) {
+            filters.add(new KeyedFilter(in));
+        }
+    }
+
+    @Override
+    protected void doWriteTo(StreamOutput out) throws IOException {
+        out.writeVInt(filters.size());
+        out.writeString(separator);
+        for (KeyedFilter keyedFilter : filters) {
+            keyedFilter.writeTo(out);
+        }
+    }
+
+    /**
+     * Set the separator used to join pairs of bucket keys
+     */
+    public AdjacencyMatrixAggregationBuilder separator(String separator) {
+        if (separator == null) {
+            throw new IllegalArgumentException("[separator] must not be null: [" + name + "]");
+        }
+        this.separator = separator;
+        return this;
+    }
+
+    /**
+     * Get the separator used to join pairs of bucket keys
+     */
+    public String separator() {
+        return separator;
+    }        
+    
+    /**
+     * Get the filters. This will be an unmodifiable map
+     */
+    public Map<String, QueryBuilder> filters() {
+        Map<String, QueryBuilder>result = new HashMap<>(this.filters.size());
+        for (KeyedFilter keyedFilter : this.filters) {
+            result.put(keyedFilter.key(), keyedFilter.filter());
+        }
+        return result;
+    }    
+    
+
+    @Override
+    protected AggregatorFactory<?> doBuild(SearchContext context, AggregatorFactory<?> parent, Builder subFactoriesBuilder)
+            throws IOException {
+        int maxFilters = context.indexShard().indexSettings().getMaxAdjacencyMatrixFilters();
+        if (filters.size() > maxFilters){
+            throw new QueryPhaseExecutionException(context,
+                    "Number of filters is too large, must be less than or equal to: [" + maxFilters + "] but was ["
+                            + filters.size() + "]."
+                            + "This limit can be set by changing the [" + IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING.getKey()
+                            + "] index level setting.");
+        }
+
+        List<KeyedFilter> rewrittenFilters = new ArrayList<>();
+        for (KeyedFilter kf : filters) {
+            rewrittenFilters.add(new KeyedFilter(kf.key(), QueryBuilder.rewriteQuery(kf.filter(), context.getQueryShardContext())));
+        }
+
+        return new AdjacencyMatrixAggregatorFactory(name, rewrittenFilters, separator, context, parent,
+                subFactoriesBuilder, metaData);
+    }
+
+    @Override
+    protected XContentBuilder internalXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(SEPARATOR_FIELD.getPreferredName(), separator);
+        builder.startObject(AdjacencyMatrixAggregator.FILTERS_FIELD.getPreferredName());
+        for (KeyedFilter keyedFilter : filters) {
+            builder.field(keyedFilter.key(), keyedFilter.filter());
+        }
+        builder.endObject();
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    protected int doHashCode() {
+        return Objects.hash(filters, separator);
+    }
+
+    @Override
+    protected boolean doEquals(Object obj) {
+        AdjacencyMatrixAggregationBuilder other = (AdjacencyMatrixAggregationBuilder) obj;
+        return Objects.equals(filters, other.filters) && Objects.equals(separator, other.separator);
+    }
+
+    @Override
+    public String getType() {
+        return NAME;
+    }
+}

core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregator.java

@@ -0,0 +1,243 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.adjacency;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.Weight;
+import org.apache.lucene.util.Bits;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.lucene.Lucene;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.ObjectParser.NamedObjectParser;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.AggregatorFactories;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.LeafBucketCollector;
+import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
+import org.elasticsearch.search.aggregations.bucket.BucketsAggregator;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.search.internal.SearchContext;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Aggregation for adjacency matrices.
+ * 
+ * NOTE! This is an experimental class.
+ * 
+ * TODO the aggregation produces a sparse response but in the
+ * computation it uses a non-sparse structure (an array of Bits
+ * objects). This could be changed to a sparse structure in future.
+ * 
+ */
+public class AdjacencyMatrixAggregator extends BucketsAggregator {
+
+    public static final ParseField FILTERS_FIELD = new ParseField("filters");
+
+    protected static class KeyedFilter implements Writeable, ToXContent {
+        private final String key;
+        private final QueryBuilder filter;
+        
+        public static final NamedObjectParser<KeyedFilter, QueryParseContext> PARSER = 
+                (XContentParser p, QueryParseContext c, String name) -> 
+                     new KeyedFilter(name, c.parseInnerQueryBuilder());
+        
+
+        public KeyedFilter(String key, QueryBuilder filter) {
+            if (key == null) {
+                throw new IllegalArgumentException("[key] must not be null");
+            }
+            if (filter == null) {
+                throw new IllegalArgumentException("[filter] must not be null");
+            }
+            this.key = key;
+            this.filter = filter;
+        }
+
+        /**
+         * Read from a stream.
+         */
+        public KeyedFilter(StreamInput in) throws IOException {
+            key = in.readString();
+            filter = in.readNamedWriteable(QueryBuilder.class);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeString(key);
+            out.writeNamedWriteable(filter);
+        }
+
+        public String key() {
+            return key;
+        }
+
+        public QueryBuilder filter() {
+            return filter;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.field(key, filter);
+            return builder;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(key, filter);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null) {
+                return false;
+            }
+            if (getClass() != obj.getClass()) {
+                return false;
+            }
+            KeyedFilter other = (KeyedFilter) obj;
+            return Objects.equals(key, other.key) && Objects.equals(filter, other.filter);
+        }
+    }
+
+    private final String[] keys;
+    private Weight[] filters;
+    private final int totalNumKeys;
+    private final int totalNumIntersections;
+    private final String separator;
+
+    public AdjacencyMatrixAggregator(String name, AggregatorFactories factories, String separator, String[] keys, 
+            Weight[] filters, SearchContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators, 
+            Map<String, Object> metaData) throws IOException {
+        super(name, factories, context, parent, pipelineAggregators, metaData);
+        this.separator = separator;
+        this.keys = keys;
+        this.filters = filters;
+        this.totalNumIntersections = ((keys.length * keys.length) - keys.length) / 2;
+        this.totalNumKeys = keys.length + totalNumIntersections;
+    }
+
+    private static class BitsIntersector implements Bits {
+        Bits a;
+        Bits b;
+
+        public BitsIntersector(Bits a, Bits b) {
+            super();
+            this.a = a;
+            this.b = b;
+        }
+
+        @Override
+        public boolean get(int index) {
+            return a.get(index) && b.get(index);
+        }
+
+        @Override
+        public int length() {
+            return Math.min(a.length(), b.length());
+        }
+
+    }
+
+    @Override
+    public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, final LeafBucketCollector sub) throws IOException {
+        // no need to provide deleted docs to the filter
+        final Bits[] bits = new Bits[filters.length + totalNumIntersections];
+        for (int i = 0; i < filters.length; ++i) {
+            bits[i] = Lucene.asSequentialAccessBits(ctx.reader().maxDoc(), filters[i].scorer(ctx));
+        }
+        // Add extra Bits for intersections
+        int pos = filters.length;
+        for (int i = 0; i < filters.length; i++) {
+            for (int j = i + 1; j < filters.length; j++) {
+                bits[pos++] = new BitsIntersector(bits[i], bits[j]);
+            }
+        }
+        assert pos == bits.length;
+        return new LeafBucketCollectorBase(sub, null) {
+            @Override
+            public void collect(int doc, long bucket) throws IOException {
+                for (int i = 0; i < bits.length; i++) {
+                    if (bits[i].get(doc)) {
+                        collectBucket(sub, doc, bucketOrd(bucket, i));
+                    }
+                }
+            }
+        };
+    }
+
+    @Override
+    public InternalAggregation buildAggregation(long owningBucketOrdinal) throws IOException {
+
+        // Buckets are ordered into groups - [keyed filters] [key1&key2 intersects]
+
+        List<InternalAdjacencyMatrix.InternalBucket> buckets = new ArrayList<>(filters.length);
+        for (int i = 0; i < keys.length; i++) {
+            long bucketOrd = bucketOrd(owningBucketOrdinal, i);
+            int docCount = bucketDocCount(bucketOrd);
+            // Empty buckets are not returned because this aggregation will commonly be used under a
+            // a date-histogram where we will look for transactions over time and can expect many
+            // empty buckets.
+            if (docCount > 0) {
+                InternalAdjacencyMatrix.InternalBucket bucket = new InternalAdjacencyMatrix.InternalBucket(keys[i], 
+                        docCount, bucketAggregations(bucketOrd));
+                buckets.add(bucket);                
+            }
+        }
+        int pos = keys.length;
+        for (int i = 0; i < keys.length; i++) {
+            for (int j = i + 1; j < keys.length; j++) {
+                long bucketOrd = bucketOrd(owningBucketOrdinal, pos);
+                int docCount = bucketDocCount(bucketOrd);
+                // Empty buckets are not returned due to potential for very sparse matrices
+                if (docCount > 0) {
+                    String intersectKey = keys[i] + separator + keys[j];
+                    InternalAdjacencyMatrix.InternalBucket bucket = new InternalAdjacencyMatrix.InternalBucket(intersectKey, 
+                            docCount, bucketAggregations(bucketOrd));
+                    buckets.add(bucket);
+                }
+                pos++;
+            }
+        }
+        return new InternalAdjacencyMatrix(name, buckets, pipelineAggregators(), metaData());
+    }
+
+    @Override
+    public InternalAggregation buildEmptyAggregation() {
+        List<InternalAdjacencyMatrix.InternalBucket> buckets = new ArrayList<>(0);
+        return new InternalAdjacencyMatrix(name, buckets, pipelineAggregators(), metaData());
+    }
+
+    final long bucketOrd(long owningBucketOrdinal, int filterOrd) {
+        return owningBucketOrdinal * totalNumKeys + filterOrd;
+    }
+
+}

core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregatorFactory.java

@@ -0,0 +1,65 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.adjacency;
+
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Weight;
+import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.AggregatorFactories;
+import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.bucket.adjacency.AdjacencyMatrixAggregator.KeyedFilter;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.search.internal.SearchContext;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+public class AdjacencyMatrixAggregatorFactory extends AggregatorFactory<AdjacencyMatrixAggregatorFactory> {
+
+    private final String[] keys;
+    private final Weight[] weights;
+    private final String separator;
+
+    public AdjacencyMatrixAggregatorFactory(String name, List<KeyedFilter> filters, String separator, 
+            SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactories, 
+            Map<String, Object> metaData) throws IOException {
+        super(name, context, parent, subFactories, metaData);
+        IndexSearcher contextSearcher = context.searcher();
+        this.separator = separator;
+        weights = new Weight[filters.size()];
+        keys = new String[filters.size()];
+        for (int i = 0; i < filters.size(); ++i) {
+            KeyedFilter keyedFilter = filters.get(i);
+            this.keys[i] = keyedFilter.key();
+            Query filter = keyedFilter.filter().toFilter(context.getQueryShardContext());
+            this.weights[i] = contextSearcher.createNormalizedWeight(filter, false);
+        }
+    }
+
+    @Override
+    public Aggregator createInternal(Aggregator parent, boolean collectsFromSingleBucket, List<PipelineAggregator> pipelineAggregators,
+            Map<String, Object> metaData) throws IOException {
+        return new AdjacencyMatrixAggregator(name, factories, separator, keys, weights, context, parent,
+                pipelineAggregators, metaData);
+    }
+
+}

core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/InternalAdjacencyMatrix.java

@@ -0,0 +1,218 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.adjacency;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class InternalAdjacencyMatrix 
+    extends InternalMultiBucketAggregation<InternalAdjacencyMatrix,InternalAdjacencyMatrix.InternalBucket> 
+    implements AdjacencyMatrix {
+    public static class InternalBucket extends InternalMultiBucketAggregation.InternalBucket implements AdjacencyMatrix.Bucket {
+
+        private final String key;
+        private long docCount;
+        InternalAggregations aggregations;
+
+        public InternalBucket(String key, long docCount, InternalAggregations aggregations) {
+            this.key = key;
+            this.docCount = docCount;
+            this.aggregations = aggregations;
+        }
+
+        /**
+         * Read from a stream.
+         */
+        public InternalBucket(StreamInput in) throws IOException {
+            key = in.readOptionalString();
+            docCount = in.readVLong();
+            aggregations = InternalAggregations.readAggregations(in);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeOptionalString(key);
+            out.writeVLong(docCount);
+            aggregations.writeTo(out);
+        }
+
+        @Override
+        public String getKey() {
+            return key;
+        }
+
+        @Override
+        public String getKeyAsString() {
+            return key;
+        }
+
+        @Override
+        public long getDocCount() {
+            return docCount;
+        }
+
+        @Override
+        public Aggregations getAggregations() {
+            return aggregations;
+        }
+
+        InternalBucket reduce(List<InternalBucket> buckets, ReduceContext context) {
+            InternalBucket reduced = null;
+            List<InternalAggregations> aggregationsList = new ArrayList<>(buckets.size());
+            for (InternalBucket bucket : buckets) {
+                if (reduced == null) {
+                    reduced = new InternalBucket(bucket.key, bucket.docCount, bucket.aggregations);
+                } else {
+                    reduced.docCount += bucket.docCount;
+                }
+                aggregationsList.add(bucket.aggregations);
+            }
+            reduced.aggregations = InternalAggregations.reduce(aggregationsList, context);
+            return reduced;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field(CommonFields.KEY, key);
+            builder.field(CommonFields.DOC_COUNT, docCount);
+            aggregations.toXContentInternal(builder, params);
+            builder.endObject();
+            return builder;
+        }
+    }
+
+    private final List<InternalBucket> buckets;
+    private Map<String, InternalBucket> bucketMap;
+
+    public InternalAdjacencyMatrix(String name, List<InternalBucket> buckets, 
+            List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
+        super(name, pipelineAggregators, metaData);
+        this.buckets = buckets;
+    }
+
+    /**
+     * Read from a stream.
+     */
+    public InternalAdjacencyMatrix(StreamInput in) throws IOException {
+        super(in);
+        int size = in.readVInt();
+        List<InternalBucket> buckets = new ArrayList<>(size);
+        for (int i = 0; i < size; i++) {
+            buckets.add(new InternalBucket(in));
+        }
+        this.buckets = buckets;
+        this.bucketMap = null;
+    }
+
+    @Override
+    protected void doWriteTo(StreamOutput out) throws IOException {
+        out.writeVInt(buckets.size());
+        for (InternalBucket bucket : buckets) {
+            bucket.writeTo(out);
+        }
+    }
+
+    @Override
+    public String getWriteableName() {
+        return AdjacencyMatrixAggregationBuilder.NAME;
+    }
+
+    @Override
+    public InternalAdjacencyMatrix create(List<InternalBucket> buckets) {
+        return new InternalAdjacencyMatrix(this.name, buckets, this.pipelineAggregators(), this.metaData);
+    }
+
+    @Override
+    public InternalBucket createBucket(InternalAggregations aggregations, InternalBucket prototype) {
+        return new InternalBucket(prototype.key, prototype.docCount, aggregations);
+    }
+
+    @Override
+    public List<InternalBucket> getBuckets() {
+        return buckets;
+    }
+
+    @Override
+    public InternalBucket getBucketByKey(String key) {
+        if (bucketMap == null) {
+            bucketMap = new HashMap<>(buckets.size());
+            for (InternalBucket bucket : buckets) {
+                bucketMap.put(bucket.getKey(), bucket);
+            }
+        }
+        return bucketMap.get(key);
+    }
+
+    @Override
+    public InternalAggregation doReduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
+        Map<String, List<InternalBucket>> bucketsMap = new HashMap<>();
+        for (InternalAggregation aggregation : aggregations) {
+            InternalAdjacencyMatrix filters = (InternalAdjacencyMatrix) aggregation;
+            for (InternalBucket bucket : filters.buckets) {
+                List<InternalBucket> sameRangeList = bucketsMap.get(bucket.key);
+                if(sameRangeList == null){
+                    sameRangeList = new ArrayList<>(aggregations.size());
+                    bucketsMap.put(bucket.key, sameRangeList);
+                }
+                sameRangeList.add(bucket);
+            }
+        }
+
+        ArrayList<InternalBucket> reducedBuckets = new ArrayList<>(bucketsMap.size());
+        for (List<InternalBucket> sameRangeList : bucketsMap.values()) {
+            InternalBucket reducedBucket = sameRangeList.get(0).reduce(sameRangeList, reduceContext);
+            if(reducedBucket.docCount >= 1){                
+                reducedBuckets.add(reducedBucket);
+            }
+        }
+        Collections.sort(reducedBuckets, Comparator.comparing(InternalBucket::getKey));
+        
+        InternalAdjacencyMatrix reduced = new InternalAdjacencyMatrix(name, reducedBuckets, pipelineAggregators(), 
+                getMetaData());
+
+        return reduced;
+    }
+
+    @Override
+    public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        builder.startArray(CommonFields.BUCKETS);
+        for (InternalBucket bucket : buckets) {
+            bucket.toXContent(builder, params);
+        }
+        builder.endArray();
+        return builder;
+    }
+
+}

core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java

@@ -288,6 +288,29 @@ public class IndexSettingsTests extends ESTestCase {
         settings = new IndexSettings(metaData, Settings.EMPTY);
         assertEquals(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY).intValue(), settings.getMaxResultWindow());
     }
+    
+    public void testMaxAdjacencyMatrixFiltersSetting() {
+        IndexMetaData metaData = newIndexMeta("index", Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put(IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING.getKey(), 15)
+            .build());
+        IndexSettings settings = new IndexSettings(metaData, Settings.EMPTY);
+        assertEquals(15, settings.getMaxAdjacencyMatrixFilters());
+        settings.updateIndexMetaData(newIndexMeta("index",
+            Settings.builder().put(IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING.getKey(),
+            42).build()));
+        assertEquals(42, settings.getMaxAdjacencyMatrixFilters());
+        settings.updateIndexMetaData(newIndexMeta("index", Settings.EMPTY));
+        assertEquals(IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING.get(Settings.EMPTY).intValue(), 
+                settings.getMaxAdjacencyMatrixFilters());
+
+        metaData = newIndexMeta("index", Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+            .build());
+        settings = new IndexSettings(metaData, Settings.EMPTY);
+        assertEquals(IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING.get(Settings.EMPTY).intValue(), 
+                settings.getMaxAdjacencyMatrixFilters());
+    }    
 
     public void testGCDeletesSetting() {
         TimeValue gcDeleteSetting = new TimeValue(Math.abs(randomInt()), TimeUnit.MILLISECONDS);

core/src/test/java/org/elasticsearch/search/aggregations/bucket/AdjacencyMatrixIT.java

@@ -0,0 +1,387 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.action.search.SearchPhaseExecutionException;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.aggregations.bucket.adjacency.AdjacencyMatrix;
+import org.elasticsearch.search.aggregations.bucket.adjacency.AdjacencyMatrix.Bucket;
+import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
+import org.elasticsearch.search.aggregations.metrics.avg.Avg;
+import org.elasticsearch.search.query.QueryPhaseExecutionException;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.hamcrest.Matchers;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
+import static org.elasticsearch.index.query.QueryBuilders.termQuery;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.adjacencyMatrix;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.avg;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.core.IsNull.notNullValue;
+
+@ESIntegTestCase.SuiteScopeTestCase
+public class AdjacencyMatrixIT extends ESIntegTestCase {
+
+    static int numDocs, numSingleTag1Docs, numSingleTag2Docs, numTag1Docs, numTag2Docs, numMultiTagDocs;
+    static final int MAX_NUM_FILTERS = 3;
+
+    @Override
+    public void setupSuiteScopeCluster() throws Exception {
+        createIndex("idx");
+        createIndex("idx2");
+        assertAcked(client().admin().indices().prepareUpdateSettings("idx")
+                .setSettings(
+                        Settings.builder().put(IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING.getKey(), MAX_NUM_FILTERS))
+                .get());
+        
+        numDocs = randomIntBetween(5, 20);
+        numTag1Docs = randomIntBetween(1, numDocs - 1);
+        numTag2Docs = randomIntBetween(1, numDocs - numTag1Docs);
+        List<IndexRequestBuilder> builders = new ArrayList<>();
+        for (int i = 0; i < numTag1Docs; i++) {
+            numSingleTag1Docs++;
+            XContentBuilder source = jsonBuilder().startObject().field("value", i + 1).field("tag", "tag1").endObject();
+            builders.add(client().prepareIndex("idx", "type", "" + i).setSource(source));
+            if (randomBoolean()) {
+                // randomly index the document twice so that we have deleted
+                // docs that match the filter
+                builders.add(client().prepareIndex("idx", "type", "" + i).setSource(source));
+            }
+        }
+        for (int i = numTag1Docs; i < (numTag1Docs + numTag2Docs); i++) {
+            numSingleTag2Docs++;
+            XContentBuilder source = jsonBuilder().startObject().field("value", i + 1).field("tag", "tag2").endObject();
+            builders.add(client().prepareIndex("idx", "type", "" + i).setSource(source));
+            if (randomBoolean()) {
+                builders.add(client().prepareIndex("idx", "type", "" + i).setSource(source));
+            }
+        }
+        for (int i = numTag1Docs + numTag2Docs; i < numDocs; i++) {
+            numMultiTagDocs++;
+            numTag1Docs++;
+            numTag2Docs++;
+            XContentBuilder source = jsonBuilder().startObject().field("value", i + 1).array("tag", "tag1", "tag2").endObject();
+            builders.add(client().prepareIndex("idx", "type", "" + i).setSource(source));
+            if (randomBoolean()) {
+                builders.add(client().prepareIndex("idx", "type", "" + i).setSource(source));
+            }
+        }
+        prepareCreate("empty_bucket_idx").addMapping("type", "value", "type=integer").execute().actionGet();
+        for (int i = 0; i < 2; i++) {
+            builders.add(client().prepareIndex("empty_bucket_idx", "type", "" + i)
+                    .setSource(jsonBuilder().startObject().field("value", i * 2).endObject()));
+        }
+        indexRandom(true, builders);
+        ensureSearchable();
+    }
+
+    public void testSimple() throws Exception {
+        SearchResponse response = client().prepareSearch("idx")
+                .addAggregation(adjacencyMatrix("tags",
+                        newMap("tag1", termQuery("tag", "tag1")).add("tag2", termQuery("tag", "tag2"))))
+                .execute().actionGet();
+
+        assertSearchResponse(response);
+
+        AdjacencyMatrix matrix = response.getAggregations().get("tags");
+        assertThat(matrix, notNullValue());
+        assertThat(matrix.getName(), equalTo("tags"));
+
+        int expected = numMultiTagDocs > 0 ? 3 : 2;
+        assertThat(matrix.getBuckets().size(), equalTo(expected));
+
+        AdjacencyMatrix.Bucket bucket = matrix.getBucketByKey("tag1");
+        assertThat(bucket, Matchers.notNullValue());
+        assertThat(bucket.getDocCount(), equalTo((long) numTag1Docs));
+
+        bucket = matrix.getBucketByKey("tag2");
+        assertThat(bucket, Matchers.notNullValue());
+        assertThat(bucket.getDocCount(), equalTo((long) numTag2Docs));
+
+        bucket = matrix.getBucketByKey("tag1&tag2");
+        if (numMultiTagDocs == 0) {
+            assertThat(bucket, Matchers.nullValue());
+        } else {
+            assertThat(bucket, Matchers.notNullValue());
+            assertThat(bucket.getDocCount(), equalTo((long) numMultiTagDocs));
+        }
+
+    }
+
+    public void testCustomSeparator() throws Exception {
+        SearchResponse response = client().prepareSearch("idx")
+                .addAggregation(adjacencyMatrix("tags", "\t",
+                        newMap("tag1", termQuery("tag", "tag1")).add("tag2", termQuery("tag", "tag2"))))
+                .execute().actionGet();
+
+        assertSearchResponse(response);
+
+        AdjacencyMatrix matrix = response.getAggregations().get("tags");
+        assertThat(matrix, notNullValue());
+
+        AdjacencyMatrix.Bucket bucket = matrix.getBucketByKey("tag1\ttag2");
+        if (numMultiTagDocs == 0) {
+            assertThat(bucket, Matchers.nullValue());
+        } else {
+            assertThat(bucket, Matchers.notNullValue());
+            assertThat(bucket.getDocCount(), equalTo((long) numMultiTagDocs));
+        }
+
+    }
+          
+
+    // See NullPointer issue when filters are empty:
+    // https://github.com/elastic/elasticsearch/issues/8438
+    public void testEmptyFilterDeclarations() throws Exception {
+        QueryBuilder emptyFilter = new BoolQueryBuilder();
+        SearchResponse response = client().prepareSearch("idx")
+                .addAggregation(adjacencyMatrix("tags",
+                        newMap("all", emptyFilter).add("tag1", termQuery("tag", "tag1"))))
+                .execute().actionGet();
+
+        assertSearchResponse(response);
+
+        AdjacencyMatrix filters = response.getAggregations().get("tags");
+        assertThat(filters, notNullValue());
+        AdjacencyMatrix.Bucket allBucket = filters.getBucketByKey("all");
+        assertThat(allBucket.getDocCount(), equalTo((long) numDocs));
+
+        AdjacencyMatrix.Bucket bucket = filters.getBucketByKey("tag1");
+        assertThat(bucket, Matchers.notNullValue());
+        assertThat(bucket.getDocCount(), equalTo((long) numTag1Docs));
+    }
+
+    public void testWithSubAggregation() throws Exception {
+        BoolQueryBuilder boolQ = new BoolQueryBuilder();
+        boolQ.must(termQuery("tag", "tag1"));
+        boolQ.must(termQuery("tag", "tag2"));
+        SearchResponse response = client().prepareSearch("idx")
+                .addAggregation(
+                        adjacencyMatrix("tags",
+                                newMap("tag1", termQuery("tag", "tag1"))
+                                    .add("tag2", termQuery("tag", "tag2")) 
+                                    .add("both", boolQ))
+                               .subAggregation(avg("avg_value").field("value")))
+                .execute().actionGet();
+
+        assertSearchResponse(response);
+
+        AdjacencyMatrix matrix = response.getAggregations().get("tags");
+        assertThat(matrix, notNullValue());
+        assertThat(matrix.getName(), equalTo("tags"));
+
+        int expectedBuckets = 0;
+        if (numTag1Docs > 0) {
+            expectedBuckets ++;
+        }
+        if (numTag2Docs > 0) {
+            expectedBuckets ++;
+        }
+        if (numMultiTagDocs > 0) {
+            // both, both&tag1, both&tag2, tag1&tag2 
+            expectedBuckets += 4;
+        }
+
+        assertThat(matrix.getBuckets().size(), equalTo(expectedBuckets));
+        assertThat(matrix.getProperty("_bucket_count"), equalTo(expectedBuckets));
+
+        Object[] propertiesKeys = (Object[]) matrix.getProperty("_key");
+        Object[] propertiesDocCounts = (Object[]) matrix.getProperty("_count");
+        Object[] propertiesCounts = (Object[]) matrix.getProperty("avg_value.value");
+
+        assertEquals(expectedBuckets, propertiesKeys.length);
+        assertEquals(propertiesKeys.length, propertiesDocCounts.length);
+        assertEquals(propertiesKeys.length, propertiesCounts.length);
+
+        for (int i = 0; i < propertiesCounts.length; i++) {
+            AdjacencyMatrix.Bucket bucket = matrix.getBucketByKey(propertiesKeys[i].toString());
+            assertThat(bucket, Matchers.notNullValue());
+            Avg avgValue = bucket.getAggregations().get("avg_value");
+            assertThat(avgValue, notNullValue());
+            assertThat((long) propertiesDocCounts[i], equalTo(bucket.getDocCount()));
+            assertThat((double) propertiesCounts[i], equalTo(avgValue.getValue()));
+        }
+
+        AdjacencyMatrix.Bucket tag1Bucket = matrix.getBucketByKey("tag1");
+        assertThat(tag1Bucket, Matchers.notNullValue());
+        assertThat(tag1Bucket.getDocCount(), equalTo((long) numTag1Docs));
+        long sum = 0;
+        for (int i = 0; i < numSingleTag1Docs; i++) {
+            sum += i + 1;
+        }
+        for (int i = numSingleTag1Docs + numSingleTag2Docs; i < numDocs; i++) {
+            sum += i + 1;
+        }
+        assertThat(tag1Bucket.getAggregations().asList().isEmpty(), is(false));
+        Avg avgBucket1Value = tag1Bucket.getAggregations().get("avg_value");
+        assertThat(avgBucket1Value, notNullValue());
+        assertThat(avgBucket1Value.getName(), equalTo("avg_value"));
+        assertThat(avgBucket1Value.getValue(), equalTo((double) sum / numTag1Docs));
+
+        Bucket tag2Bucket = matrix.getBucketByKey("tag2");
+        assertThat(tag2Bucket, Matchers.notNullValue());
+        assertThat(tag2Bucket.getDocCount(), equalTo((long) numTag2Docs));
+        sum = 0;
+        for (int i = numSingleTag1Docs; i < numDocs; i++) {
+            sum += i + 1;
+        }
+        assertThat(tag2Bucket.getAggregations().asList().isEmpty(), is(false));
+        Avg avgBucket2Value = tag2Bucket.getAggregations().get("avg_value");
+        assertThat(avgBucket2Value, notNullValue());
+        assertThat(avgBucket2Value.getName(), equalTo("avg_value"));
+        assertThat(avgBucket2Value.getValue(), equalTo((double) sum / numTag2Docs));
+
+        // Check intersection buckets are computed correctly by comparing with
+        // ANDed query bucket results
+        Bucket bucketBothQ = matrix.getBucketByKey("both");
+        if (numMultiTagDocs == 0) {
+            // Empty intersections are not returned.
+            assertThat(bucketBothQ, Matchers.nullValue());
+            Bucket bucketIntersectQ = matrix.getBucketByKey("tag1&tag2");
+            assertThat(bucketIntersectQ, Matchers.nullValue());
+            Bucket tag1Both = matrix.getBucketByKey("both&tag1");
+            assertThat(tag1Both, Matchers.nullValue());
+        } else
+        {
+            assertThat(bucketBothQ, Matchers.notNullValue());
+            assertThat(bucketBothQ.getDocCount(), equalTo((long) numMultiTagDocs));
+            Avg avgValueBothQ = bucketBothQ.getAggregations().get("avg_value");
+            
+            Bucket bucketIntersectQ = matrix.getBucketByKey("tag1&tag2");
+            assertThat(bucketIntersectQ, Matchers.notNullValue());
+            assertThat(bucketIntersectQ.getDocCount(), equalTo((long) numMultiTagDocs));
+            Avg avgValueIntersectQ = bucketBothQ.getAggregations().get("avg_value");
+            assertThat(avgValueIntersectQ.getValue(), equalTo(avgValueBothQ.getValue()));
+
+            Bucket tag1Both = matrix.getBucketByKey("both&tag1");
+            assertThat(tag1Both, Matchers.notNullValue());
+            assertThat(tag1Both.getDocCount(), equalTo((long) numMultiTagDocs));
+            Avg avgValueTag1BothIntersectQ = tag1Both.getAggregations().get("avg_value");
+            assertThat(avgValueTag1BothIntersectQ.getValue(), equalTo(avgValueBothQ.getValue()));
+        }
+
+
+    }
+    
+    public void testTooLargeMatrix() throws Exception{
+    
+        // Create more filters than is permitted by index settings.        
+        MapBuilder filtersMap = new MapBuilder();
+        for (int i = 0; i <= MAX_NUM_FILTERS; i++) {
+            filtersMap.add("tag" + i, termQuery("tag", "tag" + i));
+        }
+        
+        try {
+            client().prepareSearch("idx")
+                .addAggregation(adjacencyMatrix("tags", "\t", filtersMap))
+                .execute().actionGet();
+            fail("SearchPhaseExecutionException should have been thrown");
+        } catch (SearchPhaseExecutionException ex) {
+            assertThat(ex.getCause().getCause().getMessage(), containsString("Number of filters is too large"));
+        }        
+    }
+
+    public void testAsSubAggregation() {
+        SearchResponse response = client().prepareSearch("idx").addAggregation(histogram("histo").field("value").interval(2L)
+                .subAggregation(adjacencyMatrix("matrix", newMap("all", matchAllQuery())))).get();
+
+        assertSearchResponse(response);
+
+        Histogram histo = response.getAggregations().get("histo");
+        assertThat(histo, notNullValue());
+        assertThat(histo.getBuckets().size(), greaterThanOrEqualTo(1));
+
+        for (Histogram.Bucket bucket : histo.getBuckets()) {
+            AdjacencyMatrix matrix = bucket.getAggregations().get("matrix");
+            assertThat(matrix, notNullValue());
+            assertThat(matrix.getBuckets().size(), equalTo(1));
+            AdjacencyMatrix.Bucket filterBucket = matrix.getBuckets().get(0);
+            assertEquals(bucket.getDocCount(), filterBucket.getDocCount());
+        }
+    }
+
+    public void testWithContextBasedSubAggregation() throws Exception {
+
+        try {
+            client().prepareSearch("idx")
+                    .addAggregation(adjacencyMatrix("tags",
+                            newMap("tag1", termQuery("tag", "tag1")).add("tag2", termQuery("tag", "tag2")))
+                            .subAggregation(avg("avg_value")))
+                    .execute().actionGet();
+
+            fail("expected execution to fail - an attempt to have a context based numeric sub-aggregation, but there is not value source"
+                    + "context which the sub-aggregation can inherit");
+
+        } catch (ElasticsearchException e) {
+            assertThat(e.getMessage(), is("all shards failed"));
+        }
+    }
+
+    public void testEmptyAggregation() throws Exception {
+        SearchResponse searchResponse = client()
+                .prepareSearch("empty_bucket_idx").setQuery(matchAllQuery()).addAggregation(histogram("histo").field("value").interval(1L)
+                        .minDocCount(0).subAggregation(adjacencyMatrix("matrix", newMap("all", matchAllQuery()))))
+                .execute().actionGet();
+
+        assertThat(searchResponse.getHits().getTotalHits(), equalTo(2L));
+        Histogram histo = searchResponse.getAggregations().get("histo");
+        assertThat(histo, Matchers.notNullValue());
+        Histogram.Bucket bucket = histo.getBuckets().get(1);
+        assertThat(bucket, Matchers.notNullValue());
+
+        AdjacencyMatrix matrix = bucket.getAggregations().get("matrix");
+        assertThat(matrix, notNullValue());
+        AdjacencyMatrix.Bucket all = matrix.getBucketByKey("all");
+        assertThat(all, Matchers.nullValue());
+    }
+
+    // Helper methods for building maps of QueryBuilders
+    static MapBuilder newMap(String name, QueryBuilder builder) {
+        return new MapBuilder().add(name, builder);
+    }
+
+    static class MapBuilder extends HashMap<String, QueryBuilder> {
+        public MapBuilder add(String name, QueryBuilder builder) {
+            put(name, builder);
+            return this;
+        }
+    }
+
+}

core/src/test/java/org/elasticsearch/search/aggregations/metrics/AdjacencyMatrixTests.java

@@ -0,0 +1,59 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics;
+
+import org.elasticsearch.index.query.MatchNoneQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
+import org.elasticsearch.search.aggregations.bucket.adjacency.AdjacencyMatrixAggregationBuilder;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class AdjacencyMatrixTests extends BaseAggregationTestCase<AdjacencyMatrixAggregationBuilder> {
+
+    @Override
+    protected AdjacencyMatrixAggregationBuilder createTestAggregatorBuilder() {
+
+        int size = randomIntBetween(1, 20);
+        AdjacencyMatrixAggregationBuilder factory;
+        Map<String, QueryBuilder> filters = new HashMap<>(size);
+        for (String key : randomUnique(() -> randomAsciiOfLengthBetween(1, 20), size)) {
+            filters.put(key, QueryBuilders.termQuery(randomAsciiOfLengthBetween(5, 20), randomAsciiOfLengthBetween(5, 20)));
+        }
+        factory = new AdjacencyMatrixAggregationBuilder(randomAsciiOfLengthBetween(1, 20), filters)
+                .separator(randomFrom("&","+","\t"));       
+        return factory;
+    }
+
+    /**
+     * Test that when passing in keyed filters as a map they are equivalent
+     */
+    public void testFiltersSameMap() {
+        Map<String, QueryBuilder> original = new HashMap<>();
+        original.put("bbb", new MatchNoneQueryBuilder());
+        original.put("aaa", new MatchNoneQueryBuilder());
+        AdjacencyMatrixAggregationBuilder builder;
+        builder = new AdjacencyMatrixAggregationBuilder("my-agg", original);
+        assertEquals(original, builder.filters());
+        assert original != builder.filters();
+    }
+}

docs/reference/aggregations/bucket/adjacency-matrix-aggregation.asciidoc

@@ -0,0 +1,116 @@
+[[search-aggregations-bucket-adjacency-matrix-aggregation]]
+=== Adjacency Matrix Aggregation
+
+A bucket aggregation returning a form of https://en.wikipedia.org/wiki/Adjacency_matrix[adjacency matrix].
+The request provides a collection of named filter expressions, similar to the `filters` aggregation
+request. 
+Each bucket in the response represents a non-empty cell in the matrix of intersecting filters.
+
+experimental[The `adjacency_matrix` aggregation is a new feature and we may evolve its design as we get feedback on its use.  As a result, the API for this feature may change in non-backwards compatible ways]
+
+
+Given filters named `A`, `B` and `C` the response would return buckets with the following names:
+
+
+[options="header"]
+|=======================
+|  h|A   h|B  h|C   
+h|A |A   |A&B |A&C 
+h|B |    |B   |B&C 
+h|C |    |    |C  
+|=======================
+
+The intersecting buckets e.g `A&C` are labelled using a combination of the two filter names separated by
+the ampersand character. Note that the response does not also include a "C&A" bucket as this would be the
+same set of documents as "A&C". The matrix is said to be _symmetric_ so we only return half of it. To do this we sort 
+the filter name strings and always use the lowest of a pair as the value to the left of the "&" separator. 
+
+An alternative `separator` parameter can be passed in the request if clients wish to use a separator string 
+other than the default of the ampersand.
+
+
+Example:
+
+[source,js]
+--------------------------------------------------
+PUT /emails/message/_bulk?refresh
+{ "index" : { "_id" : 1 } }
+{ "accounts" : ["hillary", "sidney"]}
+{ "index" : { "_id" : 2 } }
+{ "accounts" : ["hillary", "donald"]}
+{ "index" : { "_id" : 3 } }
+{ "accounts" : ["vladimir", "donald"]}
+
+GET emails/message/_search
+{
+  "size": 0,
+  "aggs" : {
+    "interactions" : {
+      "adjacency_matrix" : {
+        "filters" : {
+          "grpA" : { "terms" : { "accounts" : ["hillary", "sidney"] }},
+          "grpB" : { "terms" : { "accounts" : ["donald", "mitt"] }},
+          "grpC" : { "terms" : { "accounts" : ["vladimir", "nigel"] }}
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+
+In the above example, we analyse email messages to see which groups of individuals 
+have exchanged messages.
+We will get counts for each group individually and also a count of messages for pairs
+of groups that have recorded interactions.
+
+Response:
+
+[source,js]
+--------------------------------------------------
+{
+  "took": 9,
+  "timed_out": false,
+  "_shards": ...,
+  "hits": ...,
+  "aggregations": {
+    "interactions": {
+      "buckets": [
+        {
+          "key":"grpA",
+          "doc_count": 2
+        },
+        {
+          "key":"grpA&grpB",
+          "doc_count": 1
+        },
+        {
+          "key":"grpB",
+          "doc_count": 2
+        },
+        {
+          "key":"grpB&grpC",
+          "doc_count": 1
+        },
+        {
+          "key":"grpC",
+          "doc_count": 1
+        }
+      ]
+    }
+  }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/"took": 9/"took": $body.took/]
+// TESTRESPONSE[s/"_shards": \.\.\./"_shards": $body._shards/]
+// TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/]
+
+==== Usage
+On its own this aggregation can provide all of the data required to create an undirected weighted graph.
+However, when used with child aggregations such as a `date_histogram` the results can provide the
+additional levels of data required to perform https://en.wikipedia.org/wiki/Dynamic_network_analysis[dynamic network analysis]
+where examining interactions _over time_ becomes important.
+
+==== Limitations
+For N filters the matrix of buckets produced can be N²/2 and so there is a default maximum 
+imposed of 100 filters . This setting can be changed using the `index.max_adjacency_matrix_filters` index-level setting.

rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/70_adjacency_matrix.yaml

@@ -0,0 +1,66 @@
+setup:
+  - do:
+      indices.create:
+          index: test
+          body:
+            settings:
+              number_of_shards: 1
+              number_of_replicas: 0
+            mappings:
+              post:
+                properties:
+                  num:
+                    type: integer
+
+  - do:
+        index:
+          index: test
+          type: test
+          id: 1
+          body: { "num": [1, 2] }
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 2
+        body: { "num": [2, 3] }
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 3
+        body: { "num": [3, 4] }
+
+  - do:
+      indices.refresh: {}
+
+
+---
+"Filters intersections":
+
+  - skip:
+      version: " - 5.2.99"
+      reason:  Adjacency Matrix is a 5.3.0 feature
+      
+  - do:
+      search:
+        body: { "size": 0, "aggs": { "conns": { "adjacency_matrix": {  "filters": { "1": { "term": { "num": 1 } }, "2": { "term": { "num": 2 } }, "4": { "term": { "num": 4 } } } } } } }
+
+  - match: { hits.total: 3 }
+
+  - length: { aggregations.conns.buckets: 4 }
+
+  - match: { aggregations.conns.buckets.0.doc_count: 1 }
+  - match: { aggregations.conns.buckets.0.key: "1" }
+
+  - match: { aggregations.conns.buckets.1.doc_count: 1 }
+  - match: { aggregations.conns.buckets.1.key: "1&2" }
+
+  - match: { aggregations.conns.buckets.2.doc_count: 2 }
+  - match: { aggregations.conns.buckets.2.key: "2" }
+
+  - match: { aggregations.conns.buckets.3.doc_count: 1 }
+  - match: { aggregations.conns.buckets.3.key: "4" }
+