Introduce node mappings stats (#89807)

So that they are visible in NodeIndicesStats only at the node and index (but not shard) levels. Also visible in the _cat/nodes table. And make an exact count yaml REST test.

docs/changelog/89807.yaml

@@ -0,0 +1,5 @@
+pr: 89807
+summary: Introduce node mappings stats
+area: "Stats"
+type: enhancement
+issues: []

docs/reference/cluster/nodes-stats.asciidoc

@@ -99,12 +99,14 @@ using metrics.
     metric is specified. Supported metrics are:
 +
 --
+    * `bulk`
     * `completion`
     * `docs`
     * `fielddata`
     * `flush`
     * `get`
     * `indexing`
+    * `mappings`
     * `merge`
     * `query_cache`
     * `recovery`
@@ -112,6 +114,7 @@ using metrics.
     * `request_cache`
     * `search`
     * `segments`
+    * `shard_stats`
     * `store`
     * `translog`
     * `warmer`
@@ -960,7 +963,7 @@ Time in milliseconds
 recovery operations were delayed due to throttling.
 =======
 
-`shards_stats`::
+`shard_stats`::
 (object)
 Contains statistics about all shards assigned to the node.
 +
@@ -971,6 +974,29 @@ Contains statistics about all shards assigned to the node.
 (integer)
 The total number of shards assigned to the node.
 =======
+
+`mappings`::
+(object)
+Contains statistics about the mappings for the node.
+This is not shown for the `shards` level, since mappings may be
+shared across the shards of an index on a node.
++
+.Properties of `mappings`
+[%collapsible%open]
+=======
+`total_count`::
+(integer)
+Number of mappings, including <<runtime,runtime>> and <<object,object>> fields.
+
+`total_estimated_overhead`::
+(<<byte-units,byte value>>)
+Estimated heap overhead of mappings on this node, which allows for 1kiB of heap for every mapped field.
+
+`total_estimated_overhead_in_bytes`::
+(integer)
+Estimated heap overhead, in bytes, of mappings on this node, which allows for 1kiB of heap for every mapped field.
+=======
+
 ======
 
 [[cluster-nodes-stats-api-response-body-os]]
@@ -1826,7 +1852,7 @@ The total number of kilobytes written for the device since starting {es}.
 
 `io_time_in_millis` (Linux only)::
 (integer)
-The total time in milliseconds spent performing I/O operations for the device 
+The total time in milliseconds spent performing I/O operations for the device
 since starting {es}.
 ========
 
@@ -1857,7 +1883,7 @@ since starting {es}.
 
 `io_time_in_millis` (Linux only)::
     (integer)
-    The total time in milliseconds spent performing I/O operations across all 
+    The total time in milliseconds spent performing I/O operations across all
     devices used by {es} since starting {es}.
 =======
 ======

docs/reference/how-to/size-your-shards.asciidoc

@@ -226,7 +226,11 @@ GET _cat/shards?v=true
 
 The exact resource usage of each mapped field depends on its type, but a rule
 of thumb is to allow for approximately 1kB of heap overhead per mapped field
-per index held by each data node. You must also allow enough heap for {es}'s
+per index held by each data node. In a running cluster, you can also consult the
+<<cluster-nodes-stats,Nodes stats API>>'s `mappings` indices statistic, which
+reports the number of field mappings and an estimation of their heap overhead.
+
+Additionally, you must also allow enough heap for {es}'s
 baseline usage as well as your workload such as indexing, searches and
 aggregations. 0.5GB of extra heap will suffice for many reasonable workloads,
 and you may need even less if your workload is very light while heavy workloads

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml

@@ -110,6 +110,7 @@
   - is_false:  nodes.$node_id.indices.translog
   - is_false:  nodes.$node_id.indices.recovery
   - is_false:  nodes.$node_id.indices.shard_stats
+  - is_false:  nodes.$node_id.indices.mappings
 
 ---
 "Metric - multi":
@@ -138,7 +139,8 @@
   - is_false:  nodes.$node_id.indices.segments
   - is_false:  nodes.$node_id.indices.translog
   - is_false:  nodes.$node_id.indices.recovery
-
+  - is_false:  nodes.$node_id.indices.shard_stats
+  - is_false:  nodes.$node_id.indices.mappings
 
 ---
 "Metric - recovery":
@@ -168,6 +170,7 @@
   - is_false:  nodes.$node_id.indices.translog
   - is_true:   nodes.$node_id.indices.recovery
   - is_false:  nodes.$node_id.indices.shard_stats
+  - is_false:  nodes.$node_id.indices.mappings
 
 ---
 "Metric - _all include_segment_file_sizes":
@@ -226,6 +229,7 @@
   - is_false:  nodes.$node_id.indices.translog
   - is_false:  nodes.$node_id.indices.recovery
   - is_false:  nodes.$node_id.indices.shard_stats
+  - is_false:  nodes.$node_id.indices.mappings
   - is_true:   nodes.$node_id.indices.segments.file_sizes
 
 ---
@@ -258,6 +262,7 @@
   - is_false:  nodes.$node_id.indices.translog
   - is_false:  nodes.$node_id.indices.recovery
   - is_false:  nodes.$node_id.indices.shard_stats
+  - is_false:  nodes.$node_id.indices.mappings
 
 ---
 "Metric - _all include_unloaded_segments":
@@ -388,3 +393,151 @@
       nodes.stats: { metric: _all }
 
   - gte: { nodes.$node_id.indices.shard_stats.total_count: 1 }
+
+---
+"Metric - blank for indices mappings":
+  - skip:
+      features: [arbitrary_key]
+      version: " - 8.4.99"
+      reason:  "mappings added in version 8.5.0"
+  - do:
+      nodes.info: {}
+  - set:
+      nodes._arbitrary_key_: node_id
+
+  - do:
+      nodes.stats: {}
+
+  - is_true:  nodes.$node_id.indices.mappings
+  - match: { nodes.$node_id.indices.mappings.total_count: 0 }
+  - match: { nodes.$node_id.indices.mappings.total_estimated_overhead_in_bytes: 0 }
+
+---
+"indices mappings exact count test for indices level":
+
+  - skip:
+      features: [arbitrary_key]
+      version: " - 8.4.99"
+      reason:  "mappings added in version 8.5.0"
+
+  - do:
+      indices.create:
+        index: index1
+        body:
+          settings:
+            number_of_shards: "3"
+            number_of_replicas: "1"
+          mappings:
+            runtime:
+              a_source_field:
+                type: keyword
+            properties:
+              "@timestamp":
+                type: date
+              authors:
+                properties:
+                  age:
+                    type: long
+                  company:
+                    type: text
+                    fields:
+                      keyword:
+                        type: keyword
+                        ignore_above: 256
+                  name:
+                    properties:
+                      first_name:
+                        type: keyword
+                      full_name:
+                        type: text
+                      last_name:
+                        type: keyword
+              link:
+                type: alias
+                path: url
+              title:
+                type: text
+              url:
+                type: keyword
+
+  - do:
+      nodes.info: {}
+  - set:
+      nodes._arbitrary_key_: node_id
+
+  - do:
+      nodes.stats: { metric: _all, level: "indices", human: true }
+
+  # In the below assertions, we expect a field count of 26 because the above mapping expects the following:
+  # Field mappers (incl. alias fields and object mappers' flattened leaves):
+  # 1.  _data_stream_timestamp
+  # 2.  _doc_count
+  # 3.  _feature
+  # 4. _field_names
+  # 5. _id
+  # 6. _ignored
+  # 7. _index
+  # 8. _nested_path
+  # 9. _routing
+  # 10. _seq_no
+  # 11. _source
+  # 12. _tier
+  # 13. _version
+  # 14. @timestamp
+  # 15. authors.age
+  # 16. authors.company
+  # 17. authors.company.keyword
+  # 18. authors.name.last_name
+  # 19. authors.name.first_name
+  # 20. authors.name.full_name
+  # 21. link
+  # 22. title
+  # 23. url
+  # Object mappers:
+  # 24. authors
+  # 25. authors.name
+  # Runtime field mappers:
+  # 26. a_source_field
+
+  - gte: { nodes.$node_id.indices.mappings.total_count: 26 }
+  - is_true: nodes.$node_id.indices.mappings.total_estimated_overhead
+  - gte: { nodes.$node_id.indices.mappings.total_estimated_overhead_in_bytes: 26624 }
+  - match: { nodes.$node_id.indices.indices.index1.mappings.total_count: 26 }
+  - is_true: nodes.$node_id.indices.indices.index1.mappings.total_estimated_overhead
+  - match: { nodes.$node_id.indices.indices.index1.mappings.total_estimated_overhead_in_bytes: 26624 }
+
+---
+"indices mappings does not exist in shards level":
+
+  - skip:
+      features: [arbitrary_key]
+      version: " - 8.4.99"
+      reason:  "mappings added in version 8.5.0"
+
+  - do:
+      indices.create:
+        index: index1
+        body:
+          settings:
+            number_of_shards: "3"
+            number_of_replicas: "1"
+          mappings:
+            properties:
+              prop1:
+                type: "keyword"
+              prop2:
+                type: "keyword"
+
+  - do:
+      nodes.info: {}
+  - set:
+      nodes._arbitrary_key_: node_id
+
+  - do:
+      nodes.stats: { metric: _all, level: "shards", human: true }
+
+  # We assert that there are at least two fields (for the mapping above), and that each accounts for at least 1 byte estimated overhead
+  - gte: { nodes.$node_id.indices.mappings.total_count: 2 }
+  - is_true: nodes.$node_id.indices.mappings.total_estimated_overhead
+  - gte: { nodes.$node_id.indices.mappings.total_estimated_overhead_in_bytes: 2 }
+  - is_false: nodes.$node_id.indices.shards.index1.0.mappings

server/src/internalClusterTest/java/org/elasticsearch/indices/stats/IndexStatsIT.java

@@ -844,7 +844,7 @@ public class IndexStatsIT extends ESIntegTestCase {
 
         client().admin().indices().prepareRefresh().execute().actionGet();
         IndicesStatsRequestBuilder builder = client().admin().indices().prepareStats();
-        Flag[] values = CommonStatsFlags.Flag.values();
+        Flag[] values = CommonStatsFlags.SHARD_LEVEL.getFlags();
         for (Flag flag : values) {
             set(flag, builder, false);
         }
@@ -953,7 +953,8 @@ public class IndexStatsIT extends ESIntegTestCase {
             Flag.RequestCache,
             Flag.Recovery,
             Flag.Bulk,
-            Flag.Shards };
+            Flag.Shards,
+            Flag.Mappings };
 
         assertThat(flags.length, equalTo(Flag.values().length));
         for (int i = 0; i < flags.length; i++) {
@@ -1160,6 +1161,8 @@ public class IndexStatsIT extends ESIntegTestCase {
                 return response.getBulk() != null;
             case Shards:
                 return response.getShards() != null;
+            case Mappings:
+                return response.getNodeMappings() != null;
             default:
                 fail("new flag? " + flag);
                 return false;

server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java

@@ -208,7 +208,7 @@ public class TransportClusterStatsAction extends TransportNodesAction<
                         new ShardStats(
                             indexShard.routingEntry(),
                             indexShard.shardPath(),
-                            new CommonStats(indicesService.getIndicesQueryCache(), indexShard, SHARD_STATS_FLAGS),
+                            CommonStats.getShardLevelStats(indicesService.getIndicesQueryCache(), indexShard, SHARD_STATS_FLAGS),
                             commitStats,
                             seqNoStats,
                             retentionLeaseStats

server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java

@@ -22,6 +22,7 @@ import org.elasticsearch.index.engine.SegmentsStats;
 import org.elasticsearch.index.fielddata.FieldDataStats;
 import org.elasticsearch.index.flush.FlushStats;
 import org.elasticsearch.index.get.GetStats;
+import org.elasticsearch.index.mapper.NodeMappingStats;
 import org.elasticsearch.index.merge.MergeStats;
 import org.elasticsearch.index.recovery.RecoveryStats;
 import org.elasticsearch.index.refresh.RefreshStats;
@@ -44,6 +45,8 @@ import java.util.Objects;
 
 public class CommonStats implements Writeable, ToXContentFragment {
 
+    private static final Version VERSION_SUPPORTING_NODE_MAPPINGS = Version.V_8_5_0;
+
     @Nullable
     public DocsStats docs;
 
@@ -98,6 +101,9 @@ public class CommonStats implements Writeable, ToXContentFragment {
     @Nullable
     public ShardCountStats shards;
 
+    @Nullable
+    public NodeMappingStats nodeMappings;
+
     public CommonStats() {
         this(CommonStatsFlags.NONE);
     }
@@ -125,42 +131,57 @@ public class CommonStats implements Writeable, ToXContentFragment {
                 case Recovery -> recoveryStats = new RecoveryStats();
                 case Bulk -> bulk = new BulkStats();
                 case Shards -> shards = new ShardCountStats();
+                case Mappings -> nodeMappings = new NodeMappingStats();
                 default -> throw new IllegalStateException("Unknown Flag: " + flag);
             }
         }
     }
 
-    public CommonStats(IndicesQueryCache indicesQueryCache, IndexShard indexShard, CommonStatsFlags flags) {
-        CommonStatsFlags.Flag[] setFlags = flags.getFlags();
-        for (CommonStatsFlags.Flag flag : setFlags) {
+    /**
+     * Filters the given flags for {@link CommonStatsFlags#SHARD_LEVEL} flags and calculates the corresponding statistics.
+     */
+    public static CommonStats getShardLevelStats(IndicesQueryCache indicesQueryCache, IndexShard indexShard, CommonStatsFlags flags) {
+        // Filter shard level flags
+        CommonStatsFlags filteredFlags = flags.clone();
+        for (CommonStatsFlags.Flag flag : filteredFlags.getFlags()) {
+            filteredFlags.set(flag, CommonStatsFlags.SHARD_LEVEL.isSet(flag));
+        }
+        CommonStats stats = new CommonStats(filteredFlags);
+
+        for (CommonStatsFlags.Flag flag : filteredFlags.getFlags()) {
             try {
                 switch (flag) {
-                    case Docs -> docs = indexShard.docStats();
-                    case Store -> store = indexShard.storeStats();
-                    case Indexing -> indexing = indexShard.indexingStats();
-                    case Get -> get = indexShard.getStats();
-                    case Search -> search = indexShard.searchStats(flags.groups());
-                    case Merge -> merge = indexShard.mergeStats();
-                    case Refresh -> refresh = indexShard.refreshStats();
-                    case Flush -> flush = indexShard.flushStats();
-                    case Warmer -> warmer = indexShard.warmerStats();
-                    case QueryCache -> queryCache = indicesQueryCache.getStats(indexShard.shardId());
-                    case FieldData -> fieldData = indexShard.fieldDataStats(flags.fieldDataFields());
-                    case Completion -> completion = indexShard.completionStats(flags.completionDataFields());
-                    case Segments -> segments = indexShard.segmentStats(flags.includeSegmentFileSizes(), flags.includeUnloadedSegments());
-                    case Translog -> translog = indexShard.translogStats();
-                    case RequestCache -> requestCache = indexShard.requestCache().stats();
-                    case Recovery -> recoveryStats = indexShard.recoveryStats();
-                    case Bulk -> bulk = indexShard.bulkStats();
+                    case Docs -> stats.docs = indexShard.docStats();
+                    case Store -> stats.store = indexShard.storeStats();
+                    case Indexing -> stats.indexing = indexShard.indexingStats();
+                    case Get -> stats.get = indexShard.getStats();
+                    case Search -> stats.search = indexShard.searchStats(flags.groups());
+                    case Merge -> stats.merge = indexShard.mergeStats();
+                    case Refresh -> stats.refresh = indexShard.refreshStats();
+                    case Flush -> stats.flush = indexShard.flushStats();
+                    case Warmer -> stats.warmer = indexShard.warmerStats();
+                    case QueryCache -> stats.queryCache = indicesQueryCache.getStats(indexShard.shardId());
+                    case FieldData -> stats.fieldData = indexShard.fieldDataStats(flags.fieldDataFields());
+                    case Completion -> stats.completion = indexShard.completionStats(flags.completionDataFields());
+                    case Segments -> stats.segments = indexShard.segmentStats(
+                        flags.includeSegmentFileSizes(),
+                        flags.includeUnloadedSegments()
+                    );
+                    case Translog -> stats.translog = indexShard.translogStats();
+                    case RequestCache -> stats.requestCache = indexShard.requestCache().stats();
+                    case Recovery -> stats.recoveryStats = indexShard.recoveryStats();
+                    case Bulk -> stats.bulk = indexShard.bulkStats();
                     case Shards ->
                         // Setting to 1 because the single IndexShard passed to this method implies 1 shard
-                        shards = new ShardCountStats(1);
-                    default -> throw new IllegalStateException("Unknown Flag: " + flag);
+                        stats.shards = new ShardCountStats(1);
+                    default -> throw new IllegalStateException("Unknown or invalid flag for shard-level stats: " + flag);
                 }
             } catch (AlreadyClosedException e) {
                 // shard is closed - no stats is fine
             }
         }
+
+        return stats;
     }
 
     public CommonStats(StreamInput in) throws IOException {
@@ -184,6 +205,9 @@ public class CommonStats implements Writeable, ToXContentFragment {
             bulk = in.readOptionalWriteable(BulkStats::new);
         }
         shards = in.readOptionalWriteable(ShardCountStats::new);
+        if (in.getVersion().onOrAfter(VERSION_SUPPORTING_NODE_MAPPINGS)) {
+            nodeMappings = in.readOptionalWriteable(NodeMappingStats::new);
+        }
     }
 
     @Override
@@ -208,6 +232,9 @@ public class CommonStats implements Writeable, ToXContentFragment {
             out.writeOptionalWriteable(bulk);
         }
         out.writeOptionalWriteable(shards);
+        if (out.getVersion().onOrAfter(VERSION_SUPPORTING_NODE_MAPPINGS)) {
+            out.writeOptionalWriteable(nodeMappings);
+        }
     }
 
     @Override
@@ -232,7 +259,8 @@ public class CommonStats implements Writeable, ToXContentFragment {
             && Objects.equals(requestCache, that.requestCache)
             && Objects.equals(recoveryStats, that.recoveryStats)
             && Objects.equals(bulk, that.bulk)
-            && Objects.equals(shards, that.shards);
+            && Objects.equals(shards, that.shards)
+            && Objects.equals(nodeMappings, that.nodeMappings);
     }
 
     @Override
@@ -255,7 +283,8 @@ public class CommonStats implements Writeable, ToXContentFragment {
             requestCache,
             recoveryStats,
             bulk,
-            shards
+            shards,
+            nodeMappings
         );
     }
 
@@ -404,6 +433,14 @@ public class CommonStats implements Writeable, ToXContentFragment {
                 shards = shards.add(stats.shards);
             }
         }
+        if (stats.getNodeMappings() != null) {
+            if (nodeMappings == null) {
+                nodeMappings = new NodeMappingStats();
+                nodeMappings.add(stats.getNodeMappings());
+            } else {
+                nodeMappings.add(stats.getNodeMappings());
+            }
+        }
     }
 
     @Nullable
@@ -496,6 +533,11 @@ public class CommonStats implements Writeable, ToXContentFragment {
         return shards;
     }
 
+    @Nullable
+    public NodeMappingStats getNodeMappings() {
+        return nodeMappings;
+    }
+
     /**
      * Utility method which computes total memory by adding
      * FieldData, PercolatorCache, Segments (index writer, version map)
@@ -536,6 +578,7 @@ public class CommonStats implements Writeable, ToXContentFragment {
         addIfNonNull(builder, params, requestCache);
         addIfNonNull(builder, params, recoveryStats);
         addIfNonNull(builder, params, bulk);
+        addIfNonNull(builder, params, nodeMappings);
         return builder;
     }
 

server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java

@@ -18,9 +18,16 @@ import java.io.IOException;
 import java.util.Collections;
 import java.util.EnumSet;
 
+/**
+ * Contains flags that can be used to regulate the presence and calculation of different stat fields in {@link CommonStats}.
+ *
+ * The SHARD_LEVEL flags are for stat fields that can be calculated at the shard level and then may be later aggregated at the index level
+ * along with index-level flag stat fields (e.g., Mappings).
+ */
 public class CommonStatsFlags implements Writeable, Cloneable {
 
     public static final CommonStatsFlags ALL = new CommonStatsFlags().all();
+    public static final CommonStatsFlags SHARD_LEVEL = new CommonStatsFlags().all().set(Flag.Mappings, false);
     public static final CommonStatsFlags NONE = new CommonStatsFlags().clear();
 
     private EnumSet<Flag> flags = EnumSet.allOf(Flag.class);
@@ -214,7 +221,8 @@ public class CommonStatsFlags implements Writeable, Cloneable {
         RequestCache("request_cache", 15),
         Recovery("recovery", 16),
         Bulk("bulk", 17),
-        Shards("shard_stats", 18);
+        Shards("shard_stats", 18),
+        Mappings("mappings", 19);
 
         private final String restName;
         private final int index;

server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java

@@ -120,7 +120,7 @@ public class TransportIndicesStatsAction extends TransportBroadcastByNodeAction<
                 throw new ShardNotFoundException(indexShard.shardId());
             }
 
-            CommonStats commonStats = new CommonStats(indicesService.getIndicesQueryCache(), indexShard, request.flags());
+            CommonStats commonStats = CommonStats.getShardLevelStats(indicesService.getIndicesQueryCache(), indexShard, request.flags());
             CommitStats commitStats;
             SeqNoStats seqNoStats;
             RetentionLeaseStats retentionLeaseStats;

server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java

@@ -98,7 +98,7 @@ public class Metadata extends AbstractCollection<IndexMetadata> implements Diffa
     public static final String UNKNOWN_CLUSTER_UUID = "_na_";
 
     public enum XContentContext {
-        /* Custom metadata should be returns as part of API call */
+        /* Custom metadata should be returned as part of API call */
         API,
 
         /* Custom metadata should be stored as part of the persistent cluster state */

server/src/main/java/org/elasticsearch/index/IndexService.java

@@ -51,6 +51,7 @@ import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.MapperRegistry;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.NodeMappingStats;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.index.query.SearchIndexNameMatcher;
 import org.elasticsearch.index.seqno.RetentionLeaseSyncer;
@@ -298,6 +299,14 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
         return indexShard;
     }
 
+    public NodeMappingStats getNodeMappingStats() {
+        long totalCount = mapperService().mappingLookup().getTotalFieldsCount();
+        Index index = index();
+        long totalEstimatedOverhead = totalCount * 1024L; // 1KiB estimated per mapping
+        NodeMappingStats indexNodeMappingStats = new NodeMappingStats(totalCount, totalEstimatedOverhead);
+        return indexNodeMappingStats;
+    }
+
     public Set<Integer> shardIds() {
         return shards.keySet();
     }

server/src/main/java/org/elasticsearch/index/cache/query/QueryCacheStats.java

@@ -47,6 +47,9 @@ public class QueryCacheStats implements Writeable, ToXContentFragment {
     }
 
     public void add(QueryCacheStats stats) {
+        if (stats == null) {
+            return;
+        }
         ramBytesUsed += stats.ramBytesUsed;
         hitCount += stats.hitCount;
         missCount += stats.missCount;

server/src/main/java/org/elasticsearch/index/cache/request/RequestCacheStats.java

@@ -42,6 +42,9 @@ public class RequestCacheStats implements Writeable, ToXContentFragment {
     }
 
     public void add(RequestCacheStats stats) {
+        if (stats == null) {
+            return;
+        }
         this.memorySize += stats.memorySize;
         this.evictions += stats.evictions;
         this.hitCount += stats.hitCount;

server/src/main/java/org/elasticsearch/index/fielddata/FieldDataStats.java

@@ -49,6 +49,9 @@ public class FieldDataStats implements Writeable, ToXContentFragment {
     }
 
     public void add(FieldDataStats stats) {
+        if (stats == null) {
+            return;
+        }
         this.memorySize += stats.memorySize;
         this.evictions += stats.evictions;
         if (stats.fields != null) {

server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java

@@ -220,6 +220,13 @@ public final class MappingLookup {
         return fieldTypeLookup;
     }
 
+    /**
+     * Returns the total number of fields defined in the mappings, including field mappers, object mappers as well as runtime fields.
+     */
+    public long getTotalFieldsCount() {
+        return fieldMappers.size() + objectMappers.size() + runtimeFieldMappersCount;
+    }
+
     FieldTypeLookup indexTimeLookup() {
         return indexTimeLookup;
     }
@@ -265,8 +272,7 @@ public final class MappingLookup {
     }
 
     void checkFieldLimit(long limit, int additionalFieldsToAdd) {
-        if (fieldMappers.size() + objectMappers.size() + runtimeFieldMappersCount + additionalFieldsToAdd - mapping
-            .getSortedMetadataMappers().length > limit) {
+        if (getTotalFieldsCount() + additionalFieldsToAdd - mapping.getSortedMetadataMappers().length > limit) {
             throw new IllegalArgumentException(
                 "Limit of total fields ["
                     + limit

server/src/main/java/org/elasticsearch/index/mapper/NodeMappingStats.java

@@ -0,0 +1,93 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.xcontent.ToXContentFragment;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * Node stats for mappings, useful for estimating the overhead of {@link MappingLookup} on data nodes.
+ * Should be used at node or index level, and not at shard level, since the mappings may be shared across the shards of an index.
+ */
+public class NodeMappingStats implements Writeable, ToXContentFragment {
+
+    private static final class Fields {
+        static final String MAPPINGS = "mappings";
+        static final String TOTAL_COUNT = "total_count";
+        static final String TOTAL_ESTIMATED_OVERHEAD = "total_estimated_overhead";
+        static final String TOTAL_ESTIMATED_OVERHEAD_IN_BYTES = "total_estimated_overhead_in_bytes";
+    }
+
+    private long totalCount;
+    private long totalEstimatedOverhead;
+
+    public NodeMappingStats() {
+
+    }
+
+    public NodeMappingStats(StreamInput in) throws IOException {
+        totalCount = in.readVLong();
+        totalEstimatedOverhead = in.readVLong();
+    }
+
+    public NodeMappingStats(long totalCount, long totalEstimatedOverhead) {
+        this.totalCount = totalCount;
+        this.totalEstimatedOverhead = totalEstimatedOverhead;
+    }
+
+    public void add(@Nullable NodeMappingStats other) {
+        if (other == null) return;
+        this.totalCount += other.totalCount;
+        this.totalEstimatedOverhead += other.totalEstimatedOverhead;
+    }
+
+    public long getTotalCount() {
+        return this.totalCount;
+    }
+
+    public ByteSizeValue getTotalEstimatedOverhead() {
+        return new ByteSizeValue(totalEstimatedOverhead);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVLong(totalCount);
+        out.writeVLong(totalEstimatedOverhead);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(Fields.MAPPINGS);
+        builder.field(Fields.TOTAL_COUNT, getTotalCount());
+        builder.humanReadableField(Fields.TOTAL_ESTIMATED_OVERHEAD_IN_BYTES, Fields.TOTAL_ESTIMATED_OVERHEAD, getTotalEstimatedOverhead());
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        NodeMappingStats that = (NodeMappingStats) o;
+        return totalCount == that.totalCount && totalEstimatedOverhead == that.totalEstimatedOverhead;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(totalCount, totalEstimatedOverhead);
+    }
+}

server/src/main/java/org/elasticsearch/indices/IndicesService.java

@@ -457,7 +457,24 @@ public class IndicesService extends AbstractLifecycleComponent
             }
         }
 
-        return new NodeIndicesStats(commonStats, statsByShard(this, flags));
+        return new NodeIndicesStats(commonStats, statsByIndex(this, flags), statsByShard(this, flags));
+    }
+
+    static Map<Index, CommonStats> statsByIndex(final IndicesService indicesService, final CommonStatsFlags flags) {
+        // Currently only the Mappings flag is the only possible index-level flag.
+        if (flags.isSet(CommonStatsFlags.Flag.Mappings) == false) {
+            return Map.of();
+        }
+
+        final Map<Index, CommonStats> statsByIndex = Maps.newHashMapWithExpectedSize(indicesService.indices.size());
+        for (final IndexService indexService : indicesService) {
+            Index index = indexService.index();
+            CommonStats commonStats = new CommonStats(CommonStatsFlags.NONE);
+            commonStats.nodeMappings = indexService.getNodeMappingStats();
+            var existing = statsByIndex.putIfAbsent(index, commonStats);
+            assert existing == null;
+        }
+        return statsByIndex;
     }
 
     static Map<Index, List<IndexShardStats>> statsByShard(final IndicesService indicesService, final CommonStatsFlags flags) {
@@ -512,7 +529,7 @@ public class IndicesService extends AbstractLifecycleComponent
                 new ShardStats(
                     indexShard.routingEntry(),
                     indexShard.shardPath(),
-                    new CommonStats(indicesService.getIndicesQueryCache(), indexShard, flags),
+                    CommonStats.getShardLevelStats(indicesService.getIndicesQueryCache(), indexShard, flags),
                     commitStats,
                     seqNoStats,
                     retentionLeaseStats

server/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java

@@ -8,6 +8,7 @@
 
 package org.elasticsearch.indices;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.stats.CommonStats;
 import org.elasticsearch.action.admin.indices.stats.IndexShardStats;
 import org.elasticsearch.action.admin.indices.stats.ShardStats;
@@ -23,6 +24,7 @@ import org.elasticsearch.index.engine.SegmentsStats;
 import org.elasticsearch.index.fielddata.FieldDataStats;
 import org.elasticsearch.index.flush.FlushStats;
 import org.elasticsearch.index.get.GetStats;
+import org.elasticsearch.index.mapper.NodeMappingStats;
 import org.elasticsearch.index.merge.MergeStats;
 import org.elasticsearch.index.recovery.RecoveryStats;
 import org.elasticsearch.index.refresh.RefreshStats;
@@ -49,8 +51,11 @@ import java.util.Objects;
  */
 public class NodeIndicesStats implements Writeable, ToXContentFragment {
 
+    private static final Version VERSION_SUPPORTING_STATS_BY_INDEX = Version.V_8_5_0;
+
     private final CommonStats stats;
     private final Map<Index, List<IndexShardStats>> statsByShard;
+    private final Map<Index, CommonStats> statsByIndex;
 
     public NodeIndicesStats(StreamInput in) throws IOException {
         stats = new CommonStats(in);
@@ -66,10 +71,17 @@ public class NodeIndicesStats implements Writeable, ToXContentFragment {
             }
             statsByShard.put(index, indexShardStats);
         }
+
+        if (in.getVersion().onOrAfter(VERSION_SUPPORTING_STATS_BY_INDEX)) {
+            statsByIndex = in.readMap(Index::new, CommonStats::new);
+        } else {
+            statsByIndex = new HashMap<>();
+        }
     }
 
-    public NodeIndicesStats(CommonStats oldStats, Map<Index, List<IndexShardStats>> statsByShard) {
+    public NodeIndicesStats(CommonStats oldStats, Map<Index, CommonStats> statsByIndex, Map<Index, List<IndexShardStats>> statsByShard) {
         this.statsByShard = Objects.requireNonNull(statsByShard);
+        this.statsByIndex = Objects.requireNonNull(statsByIndex);
 
         // make a total common stats from old ones and current ones
         this.stats = oldStats;
@@ -80,6 +92,9 @@ public class NodeIndicesStats implements Writeable, ToXContentFragment {
                 }
             }
         }
+        for (CommonStats indexStats : statsByIndex.values()) {
+            stats.add(indexStats);
+        }
     }
 
     @Nullable
@@ -172,10 +187,18 @@ public class NodeIndicesStats implements Writeable, ToXContentFragment {
         return stats.getShards();
     }
 
+    @Nullable
+    public NodeMappingStats getNodeMappingStats() {
+        return stats.getNodeMappings();
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         stats.writeTo(out);
         out.writeMap(statsByShard, (o, k) -> k.writeTo(o), StreamOutput::writeList);
+        if (out.getVersion().onOrAfter(VERSION_SUPPORTING_STATS_BY_INDEX)) {
+            out.writeMap(statsByIndex, (o, k) -> k.writeTo(o), (o, v) -> v.writeTo(o));
+        }
     }
 
     @Override
@@ -183,12 +206,12 @@ public class NodeIndicesStats implements Writeable, ToXContentFragment {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
         NodeIndicesStats that = (NodeIndicesStats) o;
-        return Objects.equals(stats, that.stats) && Objects.equals(statsByShard, that.statsByShard);
+        return stats.equals(that.stats) && statsByShard.equals(that.statsByShard) && statsByIndex.equals(that.statsByIndex);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(stats, statsByShard);
+        return Objects.hash(stats, statsByShard, statsByIndex);
     }
 
     @Override
@@ -206,7 +229,7 @@ public class NodeIndicesStats implements Writeable, ToXContentFragment {
         stats.toXContent(builder, params);
 
         if ("indices".equals(level)) {
-            Map<Index, CommonStats> indexStats = createStatsByIndex();
+            Map<Index, CommonStats> indexStats = createCommonStatsByIndex();
             builder.startObject(Fields.INDICES);
             for (Map.Entry<Index, CommonStats> entry : indexStats.entrySet()) {
                 builder.startObject(entry.getKey().getName());
@@ -234,8 +257,9 @@ public class NodeIndicesStats implements Writeable, ToXContentFragment {
         return builder;
     }
 
-    private Map<Index, CommonStats> createStatsByIndex() {
+    private Map<Index, CommonStats> createCommonStatsByIndex() {
         Map<Index, CommonStats> statsMap = new HashMap<>();
+
         for (Map.Entry<Index, List<IndexShardStats>> entry : statsByShard.entrySet()) {
             if (statsMap.containsKey(entry.getKey()) == false) {
                 statsMap.put(entry.getKey(), new CommonStats());
@@ -248,6 +272,10 @@ public class NodeIndicesStats implements Writeable, ToXContentFragment {
             }
         }
 
+        for (Map.Entry<Index, CommonStats> entry : statsByIndex.entrySet()) {
+            statsMap.computeIfAbsent(entry.getKey(), k -> new CommonStats()).add(entry.getValue());
+        }
+
         return statsMap;
     }
 

server/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java

@@ -33,10 +33,12 @@ import org.elasticsearch.index.engine.SegmentsStats;
 import org.elasticsearch.index.fielddata.FieldDataStats;
 import org.elasticsearch.index.flush.FlushStats;
 import org.elasticsearch.index.get.GetStats;
+import org.elasticsearch.index.mapper.NodeMappingStats;
 import org.elasticsearch.index.merge.MergeStats;
 import org.elasticsearch.index.refresh.RefreshStats;
 import org.elasticsearch.index.search.stats.SearchStats;
 import org.elasticsearch.index.shard.IndexingStats;
+import org.elasticsearch.index.shard.ShardCountStats;
 import org.elasticsearch.indices.NodeIndicesStats;
 import org.elasticsearch.monitor.fs.FsInfo;
 import org.elasticsearch.monitor.jvm.JvmInfo;
@@ -302,6 +304,18 @@ public class RestNodesAction extends AbstractCatAction {
             "alias:basi,bulkAvgSizeInBytes;default:false;text-align:right;desc:average size in bytes of shard bulk"
         );
 
+        table.addCell(
+            "shard_stats.total_count",
+            "alias:sstc,shardStatsTotalCount;default:false;text-align:right;desc:number of shards assigned"
+        );
+
+        table.addCell("mappings.total_count", "alias:mtc,mappingsTotalCount;default:false;text-align:right;desc:number of mappings");
+        table.addCell(
+            "mappings.total_estimated_overhead_in_bytes",
+            "alias:mteoi,mappingsTotalEstimatedOverheadInBytes;default:false;text-align:right;desc:estimated"
+                + " overhead in bytes of mappings"
+        );
+
         table.endHeaders();
         return table;
     }
@@ -500,6 +514,13 @@ public class RestNodesAction extends AbstractCatAction {
             table.addCell(bulkStats == null ? null : bulkStats.getAvgTime());
             table.addCell(bulkStats == null ? null : bulkStats.getAvgSizeInBytes());
 
+            ShardCountStats shardCountStats = indicesStats == null ? null : indicesStats.getShardCount();
+            table.addCell(shardCountStats == null ? null : shardCountStats.getTotalCount());
+
+            NodeMappingStats nodeMappingStats = indicesStats == null ? null : indicesStats.getNodeMappingStats();
+            table.addCell(nodeMappingStats == null ? null : nodeMappingStats.getTotalCount());
+            table.addCell(nodeMappingStats == null ? null : nodeMappingStats.getTotalEstimatedOverhead());
+
             table.endRow();
         }
 

server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java

@@ -37,6 +37,7 @@ import org.elasticsearch.index.engine.SegmentsStats;
 import org.elasticsearch.index.fielddata.FieldDataStats;
 import org.elasticsearch.index.flush.FlushStats;
 import org.elasticsearch.index.get.GetStats;
+import org.elasticsearch.index.mapper.NodeMappingStats;
 import org.elasticsearch.index.merge.MergeStats;
 import org.elasticsearch.index.recovery.RecoveryStats;
 import org.elasticsearch.index.refresh.RefreshStats;
@@ -549,7 +550,13 @@ public class NodeStatsTests extends ESTestCase {
         }
     }
 
-    private static CommonStats createCommonStats() {
+    private static CommonStats createIndexLevelCommonStats() {
+        CommonStats stats = new CommonStats(new CommonStatsFlags().clear().set(CommonStatsFlags.Flag.Mappings, true));
+        stats.nodeMappings = new NodeMappingStats(randomNonNegativeLong(), randomNonNegativeLong());
+        return stats;
+    }
+
+    private static CommonStats createShardLevelCommonStats() {
         int iota = 0;
 
         final CommonStats indicesCommonStats = new CommonStats(CommonStatsFlags.ALL);
@@ -633,7 +640,7 @@ public class NodeStatsTests extends ESTestCase {
             .resolve(shardRouting.shardId().getIndex().getUUID())
             .resolve(String.valueOf(shardRouting.shardId().id()));
         ShardPath shardPath = new ShardPath(false, path, path, shardRouting.shardId());
-        return new ShardStats(shardRouting, shardPath, createCommonStats(), null, null, null);
+        return new ShardStats(shardRouting, shardPath, createShardLevelCommonStats(), null, null, null);
     }
 
     public static NodeStats createNodeStats() {
@@ -647,6 +654,9 @@ public class NodeStatsTests extends ESTestCase {
         NodeIndicesStats nodeIndicesStats = null;
         if (frequently()) {
             final Index indexTest = new Index("test", "_na_");
+            Map<Index, CommonStats> statsByIndex = new HashMap<>();
+            statsByIndex.put(indexTest, createIndexLevelCommonStats());
+
             ShardId shardId = new ShardId(indexTest, 0);
             ShardStats shardStat = createShardStats(shardId);
             IndexShardStats shardStats = new IndexShardStats(shardId, new ShardStats[] { shardStat });
@@ -656,7 +666,7 @@ public class NodeStatsTests extends ESTestCase {
             statsByShard.put(indexTest, indexShardStats);
 
             CommonStats oldStats = new CommonStats(CommonStatsFlags.ALL);
-            nodeIndicesStats = new NodeIndicesStats(oldStats, statsByShard);
+            nodeIndicesStats = new NodeIndicesStats(oldStats, statsByIndex, statsByShard);
         }
         OsStats osStats = null;
         if (frequently()) {

server/src/test/java/org/elasticsearch/action/admin/cluster/stats/VersionStatsTests.java

@@ -115,7 +115,7 @@ public class VersionStatsTests extends AbstractWireSerializingTestCase<VersionSt
         ShardStats shardStats = new ShardStats(
             shardRouting,
             new ShardPath(false, path, path, shardRouting.shardId()),
-            new CommonStats(null, indexShard, new CommonStatsFlags(CommonStatsFlags.Flag.Store)),
+            CommonStats.getShardLevelStats(null, indexShard, new CommonStatsFlags(CommonStatsFlags.Flag.Store)),
             null,
             null,
             null

server/src/test/java/org/elasticsearch/index/mapper/NodeMappingStatsTests.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+package org.elasticsearch.index.mapper;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+public class NodeMappingStatsTests extends ESTestCase {
+
+    public void testSerialize() throws IOException {
+        NodeMappingStats stats = randomNodeMappingStats();
+        BytesStreamOutput out = new BytesStreamOutput();
+        stats.writeTo(out);
+        StreamInput input = out.bytes().streamInput();
+        NodeMappingStats read = new NodeMappingStats(input);
+        assertEquals(-1, input.read());
+        assertEquals(stats.getTotalCount(), read.getTotalCount());
+        assertEquals(stats.getTotalEstimatedOverhead(), read.getTotalEstimatedOverhead());
+    }
+
+    public void testEqualityAndHashCode() {
+        NodeMappingStats stats = randomNodeMappingStats();
+        assertEquals(stats, stats);
+        assertEquals(stats.hashCode(), stats.hashCode());
+
+        NodeMappingStats stats1 = new NodeMappingStats(1L, 2L);
+        NodeMappingStats stats2 = new NodeMappingStats(3L, 5L);
+        NodeMappingStats stats3 = new NodeMappingStats(3L, 5L);
+
+        assertNotEquals(stats1, stats2);
+        assertNotEquals(stats1, stats3);
+        assertEquals(stats2, stats3);
+    }
+
+    public void testAdd() {
+        NodeMappingStats stats1 = new NodeMappingStats(1L, 2L);
+        NodeMappingStats stats2 = new NodeMappingStats(2L, 3L);
+        NodeMappingStats stats3 = new NodeMappingStats(3L, 5L);
+
+        stats1.add(stats2);
+        assertEquals(stats1, stats3);
+        assertEquals(stats1.hashCode(), stats3.hashCode());
+    }
+
+    private static NodeMappingStats randomNodeMappingStats() {
+        long totalCount = randomIntBetween(1, 100);
+        long estimatedOverhead = totalCount * 1024;
+        return new NodeMappingStats(totalCount, estimatedOverhead);
+    }
+}

server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java

@@ -1549,7 +1549,7 @@ public class IndexShardTests extends IndexShardTestCase {
         ShardStats stats = new ShardStats(
             shard.routingEntry(),
             shard.shardPath(),
-            new CommonStats(new IndicesQueryCache(Settings.EMPTY), shard, new CommonStatsFlags()),
+            CommonStats.getShardLevelStats(new IndicesQueryCache(Settings.EMPTY), shard, new CommonStatsFlags()),
             shard.commitStats(),
             shard.seqNoStats(),
             shard.getRetentionLeaseStats()

server/src/test/java/org/elasticsearch/indices/NodeIndicesStatsTests.java

@@ -19,7 +19,7 @@ import static org.hamcrest.object.HasToString.hasToString;
 public class NodeIndicesStatsTests extends ESTestCase {
 
     public void testInvalidLevel() {
-        final NodeIndicesStats stats = new NodeIndicesStats(null, Collections.emptyMap());
+        final NodeIndicesStats stats = new NodeIndicesStats(null, Collections.emptyMap(), Collections.emptyMap());
         final String level = randomAlphaOfLength(16);
         final ToXContent.Params params = new ToXContent.MapParams(Collections.singletonMap("level", level));
         final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> stats.toXContent(null, params));

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/DataTiersUsageTransportActionTests.java

@@ -766,7 +766,7 @@ public class DataTiersUsageTransportActionTests extends ESTestCase {
                 IndexShardStats shardStats = new IndexShardStats(shardId, new ShardStats[] { shardStat });
                 indexStats.computeIfAbsent(shardId.getIndex(), k -> new ArrayList<>()).add(shardStats);
             }
-            NodeIndicesStats nodeIndexStats = new NodeIndicesStats(new CommonStats(), indexStats);
+            NodeIndicesStats nodeIndexStats = new NodeIndicesStats(new CommonStats(), Collections.emptyMap(), indexStats);
             nodeStatsList.add(mockNodeStats(node, nodeIndexStats));
         }
         return nodeStatsList;

x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsMonitoringDocTests.java

@@ -343,7 +343,7 @@ public class NodeStatsMonitoringDocTests extends BaseFilteredMonitoringDocTestCa
         segmentsStats.addBitsetMemoryInBytes(++iota);
         indicesCommonStats.getSegments().add(segmentsStats);
 
-        final NodeIndicesStats indices = new NodeIndicesStats(indicesCommonStats, emptyMap());
+        final NodeIndicesStats indices = new NodeIndicesStats(indicesCommonStats, emptyMap(), emptyMap());
 
         // Filesystem
         final FsInfo.DeviceStats ioStatsOne = new FsInfo.DeviceStats(