Implement human readable indexing pressure stats (#60022)

The indexing pressure stats do not currently have human readable
variants. This commit add human readable variants and updates the
documentation.

build.gradle

@@ -174,8 +174,8 @@ tasks.register("verifyVersions") {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = true
-final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
+boolean bwc_tests_enabled = false
+final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/60022" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

docs/reference/cluster/nodes-stats.asciidoc

@@ -2096,71 +2096,129 @@ Contains <<index-modules-indexing-pressure,indexing pressure>> statistics for th
 .Properties of `indexing_pressure`
 [%collapsible%open]
 ======
-`total`::
+`memory`::
 (object)
-Contains statistics for the cumulative indexing load since the node started.
+Contains statistics for memory consumption from indexing load.
 +
-.Properties of `<total>`
+.Properties of `<memory>`
 [%collapsible%open]
 =======
-`combined_coordinating_and_primary_bytes`::
-(integer)
-Bytes consumed by indexing requests in the coordinating or primary stage. This
-value is not the sum of coordinating_bytes and primary_bytes as a node can reuse
-the coordinating bytes if the primary stage is executed locally.
+`current`::
+(object)
+Contains statistics for current indexing load.
++
+.Properties of `<current>`
+[%collapsible%open]
+========
+`combined_coordinating_and_primary`::
+(<<byte-units,byte value>>)
+Memory consumed by indexing requests in the coordinating or primary stage. This
+value is not the sum of coordinating and primary as a node can reuse the
+coordinating memory if the primary stage is executed locally.
 
-`coordinating_bytes`::
+`combined_coordinating_and_primary_in_bytes`::
 (integer)
-Bytes consumed by indexing requests in the coordinating stage.
+Memory consumed, in bytes, by indexing requests in the coordinating or primary
+stage. This value is not the sum of coordinating and primary as a node can
+reuse the coordinating memory if the primary stage is executed locally.
 
-`primary_bytes`::
-(integer)
-Bytes consumed by indexing requests in the primary stage.
+`coordinating`::
+(<<byte-units,byte value>>)
+Memory consumed by indexing requests in the coordinating stage.
 
-`all_bytes`::
+`coordinating_in_bytes`::
 (integer)
-Bytes consumed by indexing requests in the coordinating, primary, or replica stage.
+Memory consumed, in bytes, by indexing requests in the coordinating stage.
 
-`coordinating_rejections`::
+`primary`::
+(<<byte-units,byte value>>)
+Memory consumed by indexing requests in the primary stage.
+
+`primary_in_bytes`::
 (integer)
-Number of indexing requests rejected in the coordinating stage.
+Memory consumed, in bytes, by indexing requests in the primary stage.
 
-`primary_rejections`::
+`replica`::
+(<<byte-units,byte value>>)
+Memory consumed by indexing requests in the replica stage.
+
+`replica_in_bytes`::
 (integer)
-Number of indexing requests rejected in the primary stage.
+Memory consumed, in bytes, by indexing requests in the replica stage.
 
-`replica_rejections`::
+`all`::
+(<<byte-units,byte value>>)
+Memory consumed by indexing requests in the coordinating, primary, or replica stage.
+
+`all_in_bytes`::
 (integer)
-Number of indexing requests rejected in the replica stage.
-=======
-`current`::
+Memory consumed, in bytes, by indexing requests in the coordinating, primary,
+or replica stage.
+========
+`total`::
 (object)
-Contains statistics for current indexing load.
+Contains statistics for the cumulative indexing load since the node started.
 +
-.Properties of `<current>`
+.Properties of `<total>`
 [%collapsible%open]
-=======
-`combined_coordinating_and_primary_bytes`::
+========
+`combined_coordinating_and_primary`::
+(<<byte-units,byte value>>)
+Memory consumed by indexing requests in the coordinating or primary stage. This
+value is not the sum of coordinating and primary as a node can reuse the
+coordinating memory if the primary stage is executed locally.
+
+`combined_coordinating_and_primary_in_bytes`::
+(integer)
+Memory consumed, in bytes, by indexing requests in the coordinating or primary
+stage. This value is not the sum of coordinating and primary as a node can
+reuse the coordinating memory if the primary stage is executed locally.
+
+`coordinating`::
+(<<byte-units,byte value>>)
+Memory consumed by indexing requests in the coordinating stage.
+
+`coordinating_in_bytes`::
+(integer)
+Memory consumed, in bytes, by indexing requests in the coordinating stage.
+
+`primary`::
+(<<byte-units,byte value>>)
+Memory consumed by indexing requests in the primary stage.
+
+`primary_in_bytes`::
 (integer)
-Bytes consumed by indexing requests in the coordinating or primary stage. This
-value is not the sum of coordinating_bytes and primary_bytes as a node can reuse
-the coordinating bytes if the primary stage is executed locally.
+Memory consumed, in bytes, by indexing requests in the primary stage.
+
+`replica`::
+(<<byte-units,byte value>>)
+Memory consumed by indexing requests in the replica stage.
 
-`coordinating_bytes`::
+`replica_in_bytes`::
 (integer)
-Bytes consumed by indexing requests in the coordinating stage.
+Memory consumed, in bytes, by indexing requests in the replica stage.
 
-`primary_bytes`::
+`all`::
+(<<byte-units,byte value>>)
+Memory consumed by indexing requests in the coordinating, primary, or replica stage.
+
+`all_in_bytes`::
 (integer)
-Bytes consumed by indexing requests in the primary stage.
+Memory consumed, in bytes, by indexing requests in the coordinating, primary,
+or replica stage.
 
-`replica_bytes`::
+`coordinating_rejections`::
 (integer)
-Bytes consumed by indexing requests in the replica stage.
+Number of indexing requests rejected in the coordinating stage.
 
-`all_bytes`::
+`primary_rejections`::
 (integer)
-Bytes consumed by indexing requests in the coordinating, primary, or replica stage.
+Number of indexing requests rejected in the primary stage.
+
+`replica_rejections`::
+(integer)
+Number of indexing requests rejected in the replica stage.
+========
 =======
 ======
 

qa/smoke-test-http/src/test/java/org/elasticsearch/http/IndexingPressureRestIT.java

@@ -76,17 +76,17 @@ public class IndexingPressureRestIT extends HttpSmokeTestCase {
         ArrayList<Object> values = new ArrayList<>(((Map<Object, Object>) nodeStatsMap.get("nodes")).values());
         assertThat(values.size(), equalTo(2));
         XContentTestUtils.JsonMapView node1 = new XContentTestUtils.JsonMapView((Map<String, Object>) values.get(0));
-        Integer node1CombinedBytes = node1.get("indexing_pressure.total.combined_coordinating_and_primary_bytes");
-        Integer node1PrimaryBytes = node1.get("indexing_pressure.total.primary_bytes");
-        Integer node1ReplicaBytes = node1.get("indexing_pressure.total.replica_bytes");
-        Integer node1CoordinatingRejections = node1.get("indexing_pressure.total.coordinating_rejections");
-        Integer node1PrimaryRejections = node1.get("indexing_pressure.total.primary_rejections");
+        Integer node1CombinedBytes = node1.get("indexing_pressure.memory.total.combined_coordinating_and_primary_in_bytes");
+        Integer node1PrimaryBytes = node1.get("indexing_pressure.memory.total.primary_in_bytes");
+        Integer node1ReplicaBytes = node1.get("indexing_pressure.memory.total.replica_in_bytes");
+        Integer node1CoordinatingRejections = node1.get("indexing_pressure.memory.total.coordinating_rejections");
+        Integer node1PrimaryRejections = node1.get("indexing_pressure.memory.total.primary_rejections");
         XContentTestUtils.JsonMapView node2 = new XContentTestUtils.JsonMapView((Map<String, Object>) values.get(1));
-        Integer node2IndexingBytes = node2.get("indexing_pressure.total.combined_coordinating_and_primary_bytes");
-        Integer node2PrimaryBytes = node2.get("indexing_pressure.total.primary_bytes");
-        Integer node2ReplicaBytes = node2.get("indexing_pressure.total.replica_bytes");
-        Integer node2CoordinatingRejections = node2.get("indexing_pressure.total.coordinating_rejections");
-        Integer node2PrimaryRejections = node2.get("indexing_pressure.total.primary_rejections");
+        Integer node2IndexingBytes = node2.get("indexing_pressure.memory.total.combined_coordinating_and_primary_in_bytes");
+        Integer node2PrimaryBytes = node2.get("indexing_pressure.memory.total.primary_in_bytes");
+        Integer node2ReplicaBytes = node2.get("indexing_pressure.memory.total.replica_in_bytes");
+        Integer node2CoordinatingRejections = node2.get("indexing_pressure.memory.total.coordinating_rejections");
+        Integer node2PrimaryRejections = node2.get("indexing_pressure.memory.total.primary_rejections");
 
         if (node1CombinedBytes == 0) {
             assertThat(node2IndexingBytes, greaterThan(0));
@@ -128,11 +128,11 @@ public class IndexingPressureRestIT extends HttpSmokeTestCase {
         ArrayList<Object> values2 = new ArrayList<>(((Map<Object, Object>) nodeStatsMap2.get("nodes")).values());
         assertThat(values2.size(), equalTo(2));
         XContentTestUtils.JsonMapView node1AfterRejection = new XContentTestUtils.JsonMapView((Map<String, Object>) values2.get(0));
-        node1CoordinatingRejections = node1AfterRejection.get("indexing_pressure.total.coordinating_rejections");
-        node1PrimaryRejections = node1.get("indexing_pressure.total.primary_rejections");
+        node1CoordinatingRejections = node1AfterRejection.get("indexing_pressure.memory.total.coordinating_rejections");
+        node1PrimaryRejections = node1.get("indexing_pressure.memory.total.primary_rejections");
         XContentTestUtils.JsonMapView node2AfterRejection = new XContentTestUtils.JsonMapView((Map<String, Object>) values2.get(1));
-        node2CoordinatingRejections = node2AfterRejection.get("indexing_pressure.total.coordinating_rejections");
-        node2PrimaryRejections = node2AfterRejection.get("indexing_pressure.total.primary_rejections");
+        node2CoordinatingRejections = node2AfterRejection.get("indexing_pressure.memory.total.coordinating_rejections");
+        node2PrimaryRejections = node2AfterRejection.get("indexing_pressure.memory.total.primary_rejections");
 
         if (node1CoordinatingRejections == 0) {
             assertThat(node2CoordinatingRejections, equalTo(1));

rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/50_indexing_pressure.yml

@@ -14,22 +14,18 @@
       nodes.stats:
         metric: [ indexing_pressure ]
 
-  - gte:  { nodes.$node_id.indexing_pressure.total.combined_coordinating_and_primary_bytes: 0 }
-  - gte:  { nodes.$node_id.indexing_pressure.total.coordinating_bytes: 0 }
-  - gte:  { nodes.$node_id.indexing_pressure.total.primary_bytes: 0 }
-  - gte:  { nodes.$node_id.indexing_pressure.total.replica_bytes: 0 }
-  - gte:  { nodes.$node_id.indexing_pressure.total.all_bytes: 0 }
+  - gte:  { nodes.$node_id.indexing_pressure.memory.current.combined_coordinating_and_primary_in_bytes: 0 }
+  - gte:  { nodes.$node_id.indexing_pressure.memory.current.coordinating_in_bytes: 0 }
+  - gte:  { nodes.$node_id.indexing_pressure.memory.current.primary_in_bytes: 0 }
+  - gte:  { nodes.$node_id.indexing_pressure.memory.current.replica_in_bytes: 0 }
+  - gte:  { nodes.$node_id.indexing_pressure.memory.current.all_in_bytes: 0 }
 
-  - gte:  { nodes.$node_id.indexing_pressure.total.coordinating_rejections: 0 }
-  - gte:  { nodes.$node_id.indexing_pressure.total.primary_rejections: 0 }
-  - gte:  { nodes.$node_id.indexing_pressure.total.replica_rejections: 0 }
+  - gte:  { nodes.$node_id.indexing_pressure.memory.total.combined_coordinating_and_primary_in_bytes: 0 }
+  - gte:  { nodes.$node_id.indexing_pressure.memory.total.coordinating_in_bytes: 0 }
+  - gte:  { nodes.$node_id.indexing_pressure.memory.total.primary_in_bytes: 0 }
+  - gte:  { nodes.$node_id.indexing_pressure.memory.total.replica_in_bytes: 0 }
+  - gte:  { nodes.$node_id.indexing_pressure.memory.total.all_in_bytes: 0 }
 
-  - gte:  { nodes.$node_id.indexing_pressure.current.combined_coordinating_and_primary_bytes: 0 }
-  - gte:  { nodes.$node_id.indexing_pressure.current.coordinating_bytes: 0 }
-  - gte:  { nodes.$node_id.indexing_pressure.current.primary_bytes: 0 }
-  - gte:  { nodes.$node_id.indexing_pressure.current.replica_bytes: 0 }
-  - gte:  { nodes.$node_id.indexing_pressure.current.all_bytes: 0 }
-
-# TODO:
-#
-# Change skipped version after backport
+  - gte:  { nodes.$node_id.indexing_pressure.memory.total.coordinating_rejections: 0 }
+  - gte:  { nodes.$node_id.indexing_pressure.memory.total.primary_rejections: 0 }
+  - gte:  { nodes.$node_id.indexing_pressure.memory.total.replica_rejections: 0 }

server/src/main/java/org/elasticsearch/index/stats/IndexingPressureStats.java

@@ -22,6 +22,7 @@ package org.elasticsearch.index.stats;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
@@ -93,26 +94,43 @@ public class IndexingPressureStats implements Writeable, ToXContentFragment {
         out.writeVLong(replicaRejections);
     }
 
+    private static final String COMBINED = "combined_coordinating_and_primary";
+    private static final String COMBINED_IN_BYTES = "combined_coordinating_and_primary_in_bytes";
+    private static final String COORDINATING = "coordinating";
+    private static final String COORDINATING_IN_BYTES = "coordinating_in_bytes";
+    private static final String PRIMARY = "primary";
+    private static final String PRIMARY_IN_BYTES = "primary_in_bytes";
+    private static final String REPLICA = "replica";
+    private static final String REPLICA_IN_BYTES = "replica_in_bytes";
+    private static final String ALL = "all";
+    private static final String ALL_IN_BYTES = "all_in_bytes";
+    private static final String COORDINATING_REJECTIONS = "coordinating_rejections";
+    private static final String PRIMARY_REJECTIONS = "primary_rejections";
+    private static final String REPLICA_REJECTIONS = "replica_rejections";
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject("indexing_pressure");
+        builder.startObject("memory");
+        builder.startObject("current");
+        builder.humanReadableField(COMBINED_IN_BYTES, COMBINED, new ByteSizeValue(currentCombinedCoordinatingAndPrimaryBytes));
+        builder.humanReadableField(COORDINATING_IN_BYTES, COORDINATING, new ByteSizeValue(currentCoordinatingBytes));
+        builder.humanReadableField(PRIMARY_IN_BYTES, PRIMARY, new ByteSizeValue(currentPrimaryBytes));
+        builder.humanReadableField(REPLICA_IN_BYTES, REPLICA, new ByteSizeValue(currentReplicaBytes));
+        builder.humanReadableField(ALL_IN_BYTES, ALL, new ByteSizeValue(currentReplicaBytes + currentCombinedCoordinatingAndPrimaryBytes));
+        builder.endObject();
         builder.startObject("total");
-        builder.field("combined_coordinating_and_primary_bytes", totalCombinedCoordinatingAndPrimaryBytes);
-        builder.field("coordinating_bytes", totalCoordinatingBytes);
-        builder.field("primary_bytes", totalPrimaryBytes);
-        builder.field("replica_bytes", totalReplicaBytes);
-        builder.field("all_bytes", totalReplicaBytes + totalCombinedCoordinatingAndPrimaryBytes);
-        builder.field("coordinating_rejections", coordinatingRejections);
-        builder.field("primary_rejections", primaryRejections);
-        builder.field("replica_rejections", replicaRejections);
+        builder.humanReadableField(COMBINED_IN_BYTES, COMBINED, new ByteSizeValue(totalCombinedCoordinatingAndPrimaryBytes));
+        builder.humanReadableField(COORDINATING_IN_BYTES, COORDINATING, new ByteSizeValue(totalCoordinatingBytes));
+        builder.humanReadableField(PRIMARY_IN_BYTES, PRIMARY, new ByteSizeValue(totalPrimaryBytes));
+        builder.humanReadableField(REPLICA_IN_BYTES, REPLICA, new ByteSizeValue(totalReplicaBytes));
+        builder.humanReadableField(ALL_IN_BYTES, ALL, new ByteSizeValue(totalReplicaBytes + totalCombinedCoordinatingAndPrimaryBytes));
+        builder.field(COORDINATING_REJECTIONS, coordinatingRejections);
+        builder.field(PRIMARY_REJECTIONS, primaryRejections);
+        builder.field(REPLICA_REJECTIONS, replicaRejections);
         builder.endObject();
-        builder.startObject("current");
-        builder.field("combined_coordinating_and_primary_bytes", currentCombinedCoordinatingAndPrimaryBytes);
-        builder.field("coordinating_bytes", currentCoordinatingBytes);
-        builder.field("primary_bytes", currentPrimaryBytes);
-        builder.field("replica_bytes", currentReplicaBytes);
-        builder.field("all_bytes", currentCombinedCoordinatingAndPrimaryBytes + currentReplicaBytes);
         builder.endObject();
         return builder.endObject();
     }
 }
+