Adding shard count to node stats api (#75433)

* Adding shard count to _nodes/stats api

Added a shards section to each node returned by the _nodes/stats api. Currently this new section only contains a total count of all shards on the node.

docs/reference/cluster/nodes-stats.asciidoc

@@ -959,6 +959,18 @@ Time by which recovery operations were delayed due to throttling.
 Time in milliseconds
 recovery operations were delayed due to throttling.
 =======
+
+`shards`::
+(object)
+Contains statistics about all shards assigned to the node.
++
+.Properties of `shards`
+[%collapsible%open]
+=======
+`total_count`::
+(integer)
+The total number of shards assigned to the node.
+=======
 ======
 
 [[cluster-nodes-stats-api-response-body-os]]

rest-api-spec/src/main/resources/rest-api-spec/api/nodes.stats.json

@@ -127,7 +127,8 @@
                 "segments",
                 "store",
                 "warmer",
-                "bulk"
+                "bulk",
+                "shards"
               ],
               "description":"Limit the information returned for `indices` metric to the specific index metrics. Isn't used if `indices` (or `all`) metric isn't specified."
             }
@@ -175,7 +176,8 @@
                 "segments",
                 "store",
                 "warmer",
-                "bulk"
+                "bulk",
+                "shards"
               ],
               "description":"Limit the information returned for `indices` metric to the specific index metrics. Isn't used if `indices` (or `all`) metric isn't specified."
             },

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml

@@ -109,6 +109,7 @@
   - is_false:  nodes.$node_id.indices.segments
   - is_false:  nodes.$node_id.indices.translog
   - is_false:  nodes.$node_id.indices.recovery
+  - is_false:  nodes.$node_id.indices.shards
 
 ---
 "Metric - multi":
@@ -166,6 +167,7 @@
   - is_false:  nodes.$node_id.indices.segments
   - is_false:  nodes.$node_id.indices.translog
   - is_true:   nodes.$node_id.indices.recovery
+  - is_false:  nodes.$node_id.indices.shards
 
 ---
 "Metric - _all include_segment_file_sizes":
@@ -223,6 +225,7 @@
   - is_true:   nodes.$node_id.indices.segments
   - is_false:  nodes.$node_id.indices.translog
   - is_false:  nodes.$node_id.indices.recovery
+  - is_false:  nodes.$node_id.indices.shards
   - is_true:   nodes.$node_id.indices.segments.file_sizes
 
 ---
@@ -254,6 +257,7 @@
   - is_true:   nodes.$node_id.indices.segments
   - is_false:  nodes.$node_id.indices.translog
   - is_false:  nodes.$node_id.indices.recovery
+  - is_false:  nodes.$node_id.indices.shards
 
 ---
 "Metric - _all include_unloaded_segments":
@@ -315,3 +319,72 @@
   - gte: { nodes.$node_id.http.clients.0.request_size_bytes: 0 }
   # values for clients.0.closed_time_millis, clients.0.x_forwarded_for, and clients.0.x_opaque_id are often
   # null and cannot be tested here
+
+---
+"Metric - blank for indices shards":
+  - skip:
+      features: [arbitrary_key]
+      version: " - 7.99.99"
+      reason:  "total shard count added in version 8.0"
+  - do:
+      nodes.info: {}
+  - set:
+      nodes._arbitrary_key_: node_id
+
+  - do:
+      nodes.stats: {}
+
+  - is_true:  nodes.$node_id.indices.shards
+  - match: { nodes.$node_id.indices.shards.total_count: 0 }
+
+---
+"Metric - _all for indices shards":
+  - skip:
+      features: [arbitrary_key]
+      version: " - 7.99.99"
+      reason:  "total shard count added in version 8.0"
+  - do:
+      nodes.info: {}
+  - set:
+      nodes._arbitrary_key_: node_id
+
+  - do:
+      nodes.stats: { metric: _all }
+
+  - is_true:  nodes.$node_id.indices.shards
+  - match: { nodes.$node_id.indices.shards.total_count: 0 }
+
+
+---
+"indices shards total count test":
+
+  - skip:
+      features: ["allowed_warnings", arbitrary_key]
+      version: " - 7.99.99"
+      reason:  "total shard count added in version 8.0"
+
+  - do:
+      indices.create:
+        index: index1
+        body:
+          settings:
+            number_of_shards: "5"
+            number_of_replicas: "0"
+
+  - do:
+      indices.create:
+        index: index2
+        body:
+          settings:
+            number_of_shards: "3"
+            number_of_replicas: "1"
+
+  - do:
+      nodes.info: {}
+  - set:
+      nodes._arbitrary_key_: node_id
+
+  - do:
+      nodes.stats: { metric: _all }
+
+  - gte: { nodes.$node_id.indices.shards.total_count: 1 }

server/src/internalClusterTest/java/org/elasticsearch/indices/stats/IndexStatsIT.java

@@ -735,7 +735,7 @@ public class IndexStatsIT extends ESIntegTestCase {
     public void testFlagOrdinalOrder() {
         Flag[] flags = new Flag[]{Flag.Store, Flag.Indexing, Flag.Get, Flag.Search, Flag.Merge, Flag.Flush, Flag.Refresh,
                 Flag.QueryCache, Flag.FieldData, Flag.Docs, Flag.Warmer, Flag.Completion, Flag.Segments,
-                Flag.Translog, Flag.RequestCache, Flag.Recovery, Flag.Bulk};
+                Flag.Translog, Flag.RequestCache, Flag.Recovery, Flag.Bulk, Flag.Shards};
 
         assertThat(flags.length, equalTo(Flag.values().length));
         for (int i = 0; i < flags.length; i++) {
@@ -914,6 +914,10 @@ public class IndexStatsIT extends ESIntegTestCase {
             case Bulk:
                 builder.setBulk(set);
                 break;
+            case Shards:
+                // We don't actually expose shards in IndexStats, but this test fails if it isn't handled
+                builder.request().flags().set(Flag.Shards, set);
+                break;
             default:
                 fail("new flag? " + flag);
                 break;
@@ -956,6 +960,8 @@ public class IndexStatsIT extends ESIntegTestCase {
                 return response.getRecoveryStats() != null;
             case Bulk:
                 return response.getBulk() != null;
+            case Shards:
+                return response.getShards() != null;
             default:
                 fail("new flag? " + flag);
                 return false;

server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java

@@ -32,6 +32,7 @@ import org.elasticsearch.index.search.stats.SearchStats;
 import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexingStats;
+import org.elasticsearch.index.shard.ShardCountStats;
 import org.elasticsearch.index.store.StoreStats;
 import org.elasticsearch.index.translog.TranslogStats;
 import org.elasticsearch.index.warmer.WarmerStats;
@@ -96,6 +97,9 @@ public class CommonStats implements Writeable, ToXContentFragment {
     @Nullable
     public BulkStats bulk;
 
+    @Nullable
+    public ShardCountStats shards;
+
     public CommonStats() {
         this(CommonStatsFlags.NONE);
     }
@@ -156,6 +160,9 @@ public class CommonStats implements Writeable, ToXContentFragment {
                 case Bulk:
                     bulk = new BulkStats();
                     break;
+                case Shards:
+                    shards = new ShardCountStats();
+                    break;
                 default:
                     throw new IllegalStateException("Unknown Flag: " + flag);
             }
@@ -218,6 +225,10 @@ public class CommonStats implements Writeable, ToXContentFragment {
                     case Bulk:
                         bulk = indexShard.bulkStats();
                         break;
+                    case Shards:
+                        // Setting to 1 because the single IndexShard passed to this method implies 1 shard
+                        shards = new ShardCountStats(1);
+                        break;
                     default:
                         throw new IllegalStateException("Unknown Flag: " + flag);
                 }
@@ -246,6 +257,7 @@ public class CommonStats implements Writeable, ToXContentFragment {
         recoveryStats = in.readOptionalWriteable(RecoveryStats::new);
         if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
             bulk = in.readOptionalWriteable(BulkStats::new);
+            shards = in.readOptionalWriteable(ShardCountStats::new);
         }
     }
 
@@ -269,6 +281,7 @@ public class CommonStats implements Writeable, ToXContentFragment {
         out.writeOptionalWriteable(recoveryStats);
         if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
             out.writeOptionalWriteable(bulk);
+            out.writeOptionalWriteable(shards);
         }
     }
 
@@ -410,6 +423,14 @@ public class CommonStats implements Writeable, ToXContentFragment {
         } else {
             bulk.add(stats.getBulk());
         }
+        if (stats.shards != null) {
+            if (shards == null) {
+                shards = stats.shards;
+            }
+            else {
+                shards = shards.add(stats.shards);
+            }
+        }
     }
 
     @Nullable
@@ -497,6 +518,11 @@ public class CommonStats implements Writeable, ToXContentFragment {
         return bulk;
     }
 
+    @Nullable
+    public ShardCountStats getShards() {
+        return shards;
+    }
+
     /**
      * Utility method which computes total memory by adding
      * FieldData, PercolatorCache, Segments (memory, index writer, version map)
@@ -522,7 +548,7 @@ public class CommonStats implements Writeable, ToXContentFragment {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         final Stream<ToXContent> stream = Arrays.stream(new ToXContent[] {
-            docs, store, indexing, get, search, merge, refresh, flush, warmer, queryCache,
+            docs, shards, store, indexing, get, search, merge, refresh, flush, warmer, queryCache,
             fieldData, completion, segments, translog, requestCache, recoveryStats, bulk})
             .filter(Objects::nonNull);
         for (ToXContent toXContent : ((Iterable<ToXContent>)stream::iterator)) {

server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java

@@ -217,7 +217,8 @@ public class CommonStatsFlags implements Writeable, Cloneable {
         // 14 was previously used for Suggest
         RequestCache("request_cache", 15),
         Recovery("recovery", 16),
-        Bulk("bulk", 17);
+        Bulk("bulk", 17),
+        Shards("shards", 18);
 
         private final String restName;
         private final int index;

server/src/main/java/org/elasticsearch/index/shard/ShardCountStats.java

@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.shard;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ToXContentFragment;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.core.Nullable;
+
+import java.io.IOException;
+
+public class ShardCountStats implements Writeable, ToXContentFragment {
+
+    private final long totalCount;
+
+    public ShardCountStats() {
+        totalCount = 0;
+    }
+
+    public ShardCountStats(StreamInput in) throws IOException {
+        totalCount = in.readVLong();
+    }
+
+    public ShardCountStats(long totalCount) {
+        this.totalCount = totalCount;
+    }
+
+    public long getTotalCount() {
+        return this.totalCount;
+    }
+
+    public ShardCountStats add(@Nullable ShardCountStats other) {
+        return new ShardCountStats(this.totalCount + (other == null ? 0 : other.totalCount));
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(Fields.SHARDS);
+        builder.field(Fields.TOTAL_COUNT, totalCount);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVLong(totalCount);
+    }
+
+    static final class Fields {
+        static final String SHARDS = "shards";
+        static final String TOTAL_COUNT = "total_count";
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        return (o instanceof ShardCountStats) && totalCount == ((ShardCountStats) o).totalCount;
+    }
+
+    @Override
+    public int hashCode() {
+        return Long.hashCode(totalCount);
+    }
+}

server/src/test/java/org/elasticsearch/index/shard/ShardCountStatsTest.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.shard;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.junit.Assert;
+
+class ShardCountStatsTest extends AbstractWireSerializingTestCase<ShardCountStats> {
+
+    public void testAdd() {
+        ShardCountStats shardStats1 = new ShardCountStats(5);
+        ShardCountStats shardStats2 = new ShardCountStats(8);
+        ShardCountStats combinedShardStats = shardStats1.add(shardStats2);
+        Assert.assertEquals(13, combinedShardStats.getTotalCount());
+        Assert.assertEquals(5, shardStats1.getTotalCount());
+        Assert.assertEquals(8, shardStats2.getTotalCount());
+        ShardCountStats noCountGiven = new ShardCountStats();
+        Assert.assertEquals(0, noCountGiven.getTotalCount());
+        Assert.assertEquals(8, shardStats2.add(null).getTotalCount());
+        Assert.assertEquals(8, shardStats2.getTotalCount());
+    }
+
+    @Override
+    protected Writeable.Reader<ShardCountStats> instanceReader() {
+        return ShardCountStats::new;
+    }
+
+    @Override
+    protected ShardCountStats createTestInstance() {
+        return new ShardCountStats(randomIntBetween(0, Integer.MAX_VALUE));
+    }
+}