Adds transport-only flag to always include indices in the field caps transport response (#133074)

* adds request flag 'includeIndices'

* adds CCS tests for

* Update docs/changelog/133074.yaml

* iter

* randomized test

* adds description and transient modifier to includeIndices

* iter

docs/changelog/133074.yaml

@@ -0,0 +1,6 @@
+pr: 133074
+summary: Adds transport-only flag to always include indices in the field caps transport
+  response
+area: Mapping
+type: enhancement
+issues: []

server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java

@@ -26,9 +26,12 @@ import java.util.Collection;
 import java.util.List;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.aMapWithSize;
 import static org.hamcrest.Matchers.arrayContaining;
+import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.hasSize;
 
 public class CCSFieldCapabilitiesIT extends AbstractMultiClustersTestCase {
@@ -126,4 +129,130 @@ public class CCSFieldCapabilitiesIT extends AbstractMultiClustersTestCase {
         assertThat(failures, hasSize(1));
         assertThat(failures.get(0).getIndices(), arrayContaining("remote_cluster:*"));
     }
+
+    private void populateIndices(String localIndex, String remoteIndex, String remoteClusterAlias, boolean invertLocalRemoteMappings) {
+        final Client localClient = client(LOCAL_CLUSTER);
+        final Client remoteClient = client(remoteClusterAlias);
+
+        String[] localMappings = new String[] { "timestamp", "type=date", "field1", "type=keyword", "field3", "type=keyword" };
+        String[] remoteMappings = new String[] { "timestamp", "type=date", "field2", "type=long", "field3", "type=long" };
+
+        assertAcked(
+            localClient.admin().indices().prepareCreate(localIndex).setMapping(invertLocalRemoteMappings ? remoteMappings : localMappings)
+        );
+
+        assertAcked(
+            remoteClient.admin().indices().prepareCreate(remoteIndex).setMapping(invertLocalRemoteMappings ? localMappings : remoteMappings)
+        );
+    }
+
+    public void testIncludeIndices() {
+        String localIndex = "index-local";
+        String remoteIndex = "index-remote";
+        String remoteClusterAlias = "remote_cluster";
+        populateIndices(localIndex, remoteIndex, remoteClusterAlias, false);
+        remoteIndex = String.join(":", remoteClusterAlias, remoteIndex);
+        FieldCapabilitiesResponse response = client().prepareFieldCaps(localIndex, remoteIndex)
+            .setFields("*")
+            .setIncludeIndices(true)
+            .get();
+
+        assertThat(response.getIndices(), arrayContainingInAnyOrder(localIndex, remoteIndex));
+        assertThat(response.getField("timestamp"), aMapWithSize(1));
+        assertThat(response.getField("timestamp"), hasKey("date"));
+        assertThat(response.getField("timestamp").get("date").indices(), arrayContainingInAnyOrder(localIndex, remoteIndex));
+
+        assertThat(response.getField("field1"), aMapWithSize(1));
+        assertThat(response.getField("field1"), hasKey("keyword"));
+        assertThat(response.getField("field1").get("keyword").indices(), arrayContaining(localIndex));
+
+        assertThat(response.getField("field2"), aMapWithSize(1));
+        assertThat(response.getField("field2"), hasKey("long"));
+        assertThat(response.getField("field2").get("long").indices(), arrayContaining(remoteIndex));
+
+        assertThat(response.getField("field3"), aMapWithSize(2));
+        assertThat(response.getField("field3"), hasKey("long"));
+        assertThat(response.getField("field3"), hasKey("keyword"));
+        // mapping conflict, therefore indices is always present for `field3`
+        assertThat(response.getField("field3").get("long").indices(), arrayContaining(remoteIndex));
+        assertThat(response.getField("field3").get("keyword").indices(), arrayContaining(localIndex));
+
+    }
+
+    public void testRandomIncludeIndices() {
+        String localIndex = "index-local";
+        String remoteIndex = "index-remote";
+        String remoteClusterAlias = "remote_cluster";
+        populateIndices(localIndex, remoteIndex, remoteClusterAlias, false);
+        remoteIndex = String.join(":", remoteClusterAlias, remoteIndex);
+        boolean shouldAlwaysIncludeIndices = randomBoolean();
+        FieldCapabilitiesResponse response = client().prepareFieldCaps(localIndex, remoteIndex)
+            .setFields("*")
+            .setIncludeIndices(shouldAlwaysIncludeIndices)
+            .get();
+
+        assertThat(response.getIndices(), arrayContainingInAnyOrder(localIndex, remoteIndex));
+        assertThat(response.getField("timestamp"), aMapWithSize(1));
+        assertThat(response.getField("timestamp"), hasKey("date"));
+        if (shouldAlwaysIncludeIndices) {
+            assertThat(response.getField("timestamp").get("date").indices(), arrayContainingInAnyOrder(localIndex, remoteIndex));
+        } else {
+            assertNull(response.getField("timestamp").get("date").indices());
+        }
+
+        assertThat(response.getField("field1"), aMapWithSize(1));
+        assertThat(response.getField("field1"), hasKey("keyword"));
+        if (shouldAlwaysIncludeIndices) {
+            assertThat(response.getField("field1").get("keyword").indices(), arrayContaining(localIndex));
+        } else {
+            assertNull(response.getField("field1").get("keyword").indices());
+        }
+
+        assertThat(response.getField("field2"), aMapWithSize(1));
+        assertThat(response.getField("field2"), hasKey("long"));
+        if (shouldAlwaysIncludeIndices) {
+            assertThat(response.getField("field2").get("long").indices(), arrayContaining(remoteIndex));
+        } else {
+            assertNull(response.getField("field2").get("long").indices());
+        }
+
+        assertThat(response.getField("field3"), aMapWithSize(2));
+        assertThat(response.getField("field3"), hasKey("long"));
+        assertThat(response.getField("field3"), hasKey("keyword"));
+        // mapping conflict, therefore indices is always present for `field3`
+        assertThat(response.getField("field3").get("long").indices(), arrayContaining(remoteIndex));
+        assertThat(response.getField("field3").get("keyword").indices(), arrayContaining(localIndex));
+    }
+
+    public void testIncludeIndicesSwapped() {
+        // exact same setup as testIncludeIndices but with mappings swapped between local and remote index
+        String localIndex = "index-local";
+        String remoteIndex = "index-remote";
+        String remoteClusterAlias = "remote_cluster";
+        populateIndices(localIndex, remoteIndex, remoteClusterAlias, true);
+        remoteIndex = String.join(":", remoteClusterAlias, remoteIndex);
+        FieldCapabilitiesResponse response = client().prepareFieldCaps(localIndex, remoteIndex)
+            .setFields("*")
+            .setIncludeIndices(true)
+            .get();
+
+        assertThat(response.getIndices(), arrayContainingInAnyOrder(localIndex, remoteIndex));
+        assertThat(response.getField("timestamp"), aMapWithSize(1));
+        assertThat(response.getField("timestamp"), hasKey("date"));
+        assertThat(response.getField("timestamp").get("date").indices(), arrayContainingInAnyOrder(localIndex, remoteIndex));
+
+        assertThat(response.getField("field1"), aMapWithSize(1));
+        assertThat(response.getField("field1"), hasKey("keyword"));
+        assertThat(response.getField("field1").get("keyword").indices(), arrayContaining(remoteIndex));
+
+        assertThat(response.getField("field2"), aMapWithSize(1));
+        assertThat(response.getField("field2"), hasKey("long"));
+        assertThat(response.getField("field2").get("long").indices(), arrayContaining(localIndex));
+
+        assertThat(response.getField("field3"), aMapWithSize(2));
+        assertThat(response.getField("field3"), hasKey("long"));
+        assertThat(response.getField("field3"), hasKey("keyword"));
+        assertThat(response.getField("field3").get("long").indices(), arrayContaining(localIndex));
+        assertThat(response.getField("field3").get("keyword").indices(), arrayContaining(remoteIndex));
+    }
 }

server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java

@@ -93,6 +93,7 @@ import static org.elasticsearch.index.shard.IndexShardTestCase.closeShardNoCheck
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.aMapWithSize;
 import static org.hamcrest.Matchers.array;
+import static org.hamcrest.Matchers.arrayContaining;
 import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.empty;
@@ -524,6 +525,88 @@ public class FieldCapabilitiesIT extends ESIntegTestCase {
         }
     }
 
+    private void populateIndices() throws Exception {
+        internalCluster().ensureAtLeastNumDataNodes(2);
+        assertAcked(
+            prepareCreate("index-1").setSettings(indexSettings(between(1, 5), 1))
+                .setMapping("timestamp", "type=date", "field1", "type=keyword", "field3", "type=keyword"),
+            prepareCreate("index-2").setSettings(indexSettings(between(1, 5), 1))
+                .setMapping("timestamp", "type=date", "field2", "type=long", "field3", "type=long")
+        );
+    }
+
+    public void testIncludeIndices() throws Exception {
+        populateIndices();
+        FieldCapabilitiesRequest request = new FieldCapabilitiesRequest();
+        request.indices("index-*");
+        request.fields("*");
+        request.includeIndices(true);
+
+        final FieldCapabilitiesResponse response = client().execute(TransportFieldCapabilitiesAction.TYPE, request).actionGet();
+        assertThat(response.getIndices(), arrayContainingInAnyOrder("index-1", "index-2"));
+        assertThat(response.getField("timestamp"), aMapWithSize(1));
+        assertThat(response.getField("timestamp"), hasKey("date"));
+        assertThat(response.getField("timestamp").get("date").indices(), arrayContainingInAnyOrder("index-1", "index-2"));
+
+        assertThat(response.getField("field1"), aMapWithSize(1));
+        assertThat(response.getField("field1"), hasKey("keyword"));
+        assertThat(response.getField("field1").get("keyword").indices(), arrayContaining("index-1"));
+
+        assertThat(response.getField("field2"), aMapWithSize(1));
+        assertThat(response.getField("field2"), hasKey("long"));
+        assertThat(response.getField("field2").get("long").indices(), arrayContaining("index-2"));
+
+        assertThat(response.getField("field3"), aMapWithSize(2));
+        assertThat(response.getField("field3"), hasKey("long"));
+        assertThat(response.getField("field3"), hasKey("keyword"));
+        assertThat(response.getField("field3").get("long").indices(), arrayContaining("index-2"));
+        assertThat(response.getField("field3").get("keyword").indices(), arrayContaining("index-1"));
+
+    }
+
+    public void testRandomIncludeIndices() throws Exception {
+        populateIndices();
+        FieldCapabilitiesRequest request = new FieldCapabilitiesRequest();
+        request.indices("index-*");
+        request.fields("*");
+        boolean shouldAlwaysIncludeIndices = randomBoolean();
+        request.includeIndices(shouldAlwaysIncludeIndices);
+
+        final FieldCapabilitiesResponse response = client().execute(TransportFieldCapabilitiesAction.TYPE, request).actionGet();
+        assertThat(response.getIndices(), arrayContainingInAnyOrder("index-1", "index-2"));
+        assertThat(response.getField("timestamp"), aMapWithSize(1));
+        assertThat(response.getField("timestamp"), hasKey("date"));
+        if (shouldAlwaysIncludeIndices) {
+            assertThat(response.getField("timestamp").get("date").indices(), arrayContainingInAnyOrder("index-1", "index-2"));
+        } else {
+            assertNull(response.getField("timestamp").get("date").indices());
+        }
+
+        assertThat(response.getField("field1"), aMapWithSize(1));
+        assertThat(response.getField("field1"), hasKey("keyword"));
+        if (shouldAlwaysIncludeIndices) {
+            assertThat(response.getField("field1").get("keyword").indices(), arrayContaining("index-1"));
+        } else {
+            assertNull(response.getField("field1").get("keyword").indices());
+        }
+
+        assertThat(response.getField("field2"), aMapWithSize(1));
+        assertThat(response.getField("field2"), hasKey("long"));
+        if (shouldAlwaysIncludeIndices) {
+            assertThat(response.getField("field2").get("long").indices(), arrayContaining("index-2"));
+        } else {
+            assertNull(response.getField("field2").get("long").indices());
+        }
+
+        assertThat(response.getField("field3"), aMapWithSize(2));
+        assertThat(response.getField("field3"), hasKey("long"));
+        assertThat(response.getField("field3"), hasKey("keyword"));
+        // mapping conflict, therefore indices is always present for `field3`
+        assertThat(response.getField("field3").get("long").indices(), arrayContaining("index-2"));
+        assertThat(response.getField("field3").get("keyword").indices(), arrayContaining("index-1"));
+
+    }
+
     public void testNoActiveCopy() throws Exception {
         assertAcked(
             prepareCreate("log-index-inactive").setSettings(

server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java

@@ -80,8 +80,13 @@ public class FieldCapabilities implements Writeable, ToXContentObject {
      * @param isAggregatable Whether this field can be aggregated on.
      * @param isDimension Whether this field can be used as dimension
      * @param metricType If this field is a metric field, returns the metric's type or null for non-metrics fields
-     * @param indices The list of indices where this field name is defined as {@code type},
-     *                or null if all indices have the same {@code type} for the field.
+     * @param indices The list of indices where this field name is defined as {@code type}.
+     *                When {@code includeIndices} is set to {@code false}, this list is only
+     *                present if there is a mapping conflict (e.g. the same field has different
+     *                types across indices).
+     *                When {@code includeIndices} is set to {@code true}, this list is always
+     *                present and contains all indices where the field exists, regardless of
+     *                mapping conflicts.
      * @param nonSearchableIndices The list of indices where this field is not searchable,
      *                             or null if the field is searchable in all indices.
      * @param nonAggregatableIndices The list of indices where this field is not aggregatable,

server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java

@@ -53,6 +53,13 @@ public final class FieldCapabilitiesRequest extends LegacyActionRequest implemen
     private String[] types = Strings.EMPTY_ARRAY;
     private boolean includeUnmapped = false;
     private boolean includeEmptyFields = true;
+    /**
+     * Controls whether the field caps response should always include the list of indices
+     * where a field is defined. This flag is only used locally on the coordinating node,
+     * and does not need to be serialized as the indices information is already carried
+     * in the response if required.
+     */
+    private transient boolean includeIndices = false;
     // pkg private API mainly for cross cluster search to signal that we do multiple reductions ie. the results should not be merged
     private boolean mergeResults = true;
     private QueryBuilder indexFilter;
@@ -208,6 +215,11 @@ public final class FieldCapabilitiesRequest extends LegacyActionRequest implemen
         return this;
     }
 
+    public FieldCapabilitiesRequest includeIndices(boolean includeIndices) {
+        this.includeIndices = includeIndices;
+        return this;
+    }
+
     @Override
     public String[] indices() {
         return indices;
@@ -232,6 +244,10 @@ public final class FieldCapabilitiesRequest extends LegacyActionRequest implemen
         return includeUnmapped;
     }
 
+    public boolean includeIndices() {
+        return includeIndices;
+    }
+
     public boolean includeEmptyFields() {
         return includeEmptyFields;
     }

server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java

@@ -53,4 +53,9 @@ public class FieldCapabilitiesRequestBuilder extends ActionRequestBuilder<FieldC
         request().runtimeFields(runtimeFieldSection);
         return this;
     }
+
+    public FieldCapabilitiesRequestBuilder setIncludeIndices(boolean includeIndices) {
+        request().includeIndices(includeIndices);
+        return this;
+    }
 }

server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java

@@ -477,9 +477,9 @@ public class TransportFieldCapabilitiesAction extends HandledTransportAction<Fie
         task.ensureNotCancelled();
         Map<String, Map<String, FieldCapabilities>> fields = Maps.newMapWithExpectedSize(fieldsBuilder.size());
         if (request.includeUnmapped()) {
-            collectFieldsIncludingUnmapped(indices, fieldsBuilder, fields);
+            collectFieldsIncludingUnmapped(indices, fieldsBuilder, fields, request.includeIndices());
         } else {
-            collectFields(fieldsBuilder, fields);
+            collectFields(fieldsBuilder, fields, request.includeIndices());
         }
 
         // The merge method is only called on the primary coordinator for cross-cluster field caps, so we
@@ -509,7 +509,8 @@ public class TransportFieldCapabilitiesAction extends HandledTransportAction<Fie
     private static void collectFieldsIncludingUnmapped(
         String[] indices,
         Map<String, Map<String, FieldCapabilities.Builder>> fieldsBuilder,
-        Map<String, Map<String, FieldCapabilities>> fieldsMap
+        Map<String, Map<String, FieldCapabilities>> fieldsMap,
+        boolean includeIndices
     ) {
         final Set<String> mappedScratch = new HashSet<>();
         for (Map.Entry<String, Map<String, FieldCapabilities.Builder>> entry : fieldsBuilder.entrySet()) {
@@ -523,7 +524,7 @@ public class TransportFieldCapabilitiesAction extends HandledTransportAction<Fie
             var unmapped = getUnmappedFields(indices, entry.getKey(), mappedScratch);
 
             final int resSize = typeMapBuilder.size() + (unmapped == null ? 0 : 1);
-            final Map<String, FieldCapabilities> res = capabilities(resSize, typeMapBuilder);
+            final Map<String, FieldCapabilities> res = capabilities(resSize, typeMapBuilder, includeIndices);
             if (unmapped != null) {
                 res.put("unmapped", unmapped.apply(resSize > 1));
             }
@@ -533,19 +534,25 @@ public class TransportFieldCapabilitiesAction extends HandledTransportAction<Fie
 
     private static void collectFields(
         Map<String, Map<String, FieldCapabilities.Builder>> fieldsBuilder,
-        Map<String, Map<String, FieldCapabilities>> fields
+        Map<String, Map<String, FieldCapabilities>> fields,
+        boolean includeIndices
     ) {
         for (Map.Entry<String, Map<String, FieldCapabilities.Builder>> entry : fieldsBuilder.entrySet()) {
             var typeMapBuilder = entry.getValue().entrySet();
-            fields.put(entry.getKey(), Collections.unmodifiableMap(capabilities(typeMapBuilder.size(), typeMapBuilder)));
+            fields.put(entry.getKey(), Collections.unmodifiableMap(capabilities(typeMapBuilder.size(), typeMapBuilder, includeIndices)));
         }
     }
 
-    private static Map<String, FieldCapabilities> capabilities(int resSize, Set<Map.Entry<String, FieldCapabilities.Builder>> builders) {
+    private static Map<String, FieldCapabilities> capabilities(
+        int resSize,
+        Set<Map.Entry<String, FieldCapabilities.Builder>> builders,
+        boolean includeIndices
+    ) {
         boolean multiTypes = resSize > 1;
+        boolean withIndices = multiTypes || includeIndices;
         final Map<String, FieldCapabilities> res = Maps.newHashMapWithExpectedSize(resSize);
         for (Map.Entry<String, FieldCapabilities.Builder> e : builders) {
-            res.put(e.getKey(), e.getValue().build(multiTypes));
+            res.put(e.getKey(), e.getValue().build(withIndices));
         }
         return res;
     }