Merge pull request #20255 from javanna/enhancement/cluster_stats_available_memory

Add mem section back to cluster stats

core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java

@@ -212,7 +212,7 @@ public class NodeStats extends BaseNodeResponse implements ToXContent {
             indices = NodeIndicesStats.readIndicesStats(in);
         }
         if (in.readBoolean()) {
-            os = OsStats.readOsStats(in);
+            os = new OsStats(in);
         }
         if (in.readBoolean()) {
             process = ProcessStats.readProcessStats(in);

core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java

@@ -87,7 +87,7 @@ public class ClusterStatsNodes implements ToXContent {
             }
         }
         this.counts = new Counts(nodeInfos);
-        this.os = new OsStats(nodeInfos);
+        this.os = new OsStats(nodeInfos, nodeStats);
         this.process = new ProcessStats(nodeStats);
         this.jvm = new JvmStats(nodeInfos, nodeStats);
         this.networkTypes = new NetworkTypes(nodeInfos);
@@ -226,11 +226,12 @@ public class ClusterStatsNodes implements ToXContent {
         final int availableProcessors;
         final int allocatedProcessors;
         final ObjectIntHashMap<String> names;
+        final org.elasticsearch.monitor.os.OsStats.Mem mem;
 
         /**
          * Build the stats from information about each node.
          */
-        private OsStats(List<NodeInfo> nodeInfos) {
+        private OsStats(List<NodeInfo> nodeInfos, List<NodeStats> nodeStatsList) {
             this.names = new ObjectIntHashMap<>();
             int availableProcessors = 0;
             int allocatedProcessors = 0;
@@ -244,6 +245,22 @@ public class ClusterStatsNodes implements ToXContent {
             }
             this.availableProcessors = availableProcessors;
             this.allocatedProcessors = allocatedProcessors;
+
+            long totalMemory = 0;
+            long freeMemory = 0;
+            for (NodeStats nodeStats : nodeStatsList) {
+                if (nodeStats.getOs() != null) {
+                    long total = nodeStats.getOs().getMem().getTotal().bytes();
+                    if (total > 0) {
+                        totalMemory += total;
+                    }
+                    long free = nodeStats.getOs().getMem().getFree().bytes();
+                    if (free > 0) {
+                        freeMemory += free;
+                    }
+                }
+            }
+            this.mem = new org.elasticsearch.monitor.os.OsStats.Mem(totalMemory, freeMemory);
         }
 
         public int getAvailableProcessors() {
@@ -254,6 +271,10 @@ public class ClusterStatsNodes implements ToXContent {
             return allocatedProcessors;
         }
 
+        public org.elasticsearch.monitor.os.OsStats.Mem getMem() {
+            return mem;
+        }
+
         static final class Fields {
             static final String AVAILABLE_PROCESSORS = "available_processors";
             static final String ALLOCATED_PROCESSORS = "allocated_processors";
@@ -274,6 +295,7 @@ public class ClusterStatsNodes implements ToXContent {
                 builder.endObject();
             }
             builder.endArray();
+            mem.toXContent(builder, params);
             return builder;
         }
     }

core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java

@@ -92,7 +92,7 @@ public class TransportClusterStatsAction extends TransportNodesAction<ClusterSta
     @Override
     protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeRequest) {
         NodeInfo nodeInfo = nodeService.info(true, true, false, true, false, true, false, true, false, false);
-        NodeStats nodeStats = nodeService.stats(CommonStatsFlags.NONE, false, true, true, false, true, false, false, false, false, false, false);
+        NodeStats nodeStats = nodeService.stats(CommonStatsFlags.NONE, true, true, true, false, true, false, false, false, false, false, false);
         List<ShardStats> shardsStats = new ArrayList<>();
         for (IndexService indexService : indicesService) {
             for (IndexShard indexShard : indexService) {

core/src/main/java/org/elasticsearch/monitor/os/OsProbe.java

@@ -173,23 +173,10 @@ public class OsProbe {
     }
 
     public OsStats osStats() {
-        OsStats stats = new OsStats();
-        stats.timestamp = System.currentTimeMillis();
-        stats.cpu = new OsStats.Cpu();
-        stats.cpu.percent = getSystemCpuPercent();
-        stats.cpu.loadAverage = getSystemLoadAverage();
-
-        OsStats.Mem mem = new OsStats.Mem();
-        mem.total = getTotalPhysicalMemorySize();
-        mem.free = getFreePhysicalMemorySize();
-        stats.mem = mem;
-
-        OsStats.Swap swap = new OsStats.Swap();
-        swap.total = getTotalSwapSpaceSize();
-        swap.free = getFreeSwapSpaceSize();
-        stats.swap = swap;
-
-        return stats;
+        OsStats.Cpu cpu = new OsStats.Cpu(getSystemCpuPercent(), getSystemLoadAverage());
+        OsStats.Mem mem = new OsStats.Mem(getTotalPhysicalMemorySize(), getFreePhysicalMemorySize());
+        OsStats.Swap swap = new OsStats.Swap(getTotalSwapSpaceSize(), getFreeSwapSpaceSize());
+        return new OsStats(System.currentTimeMillis(), cpu, mem , swap);
     }
 
     /**

core/src/main/java/org/elasticsearch/monitor/os/OsStats.java

@@ -21,28 +21,42 @@ package org.elasticsearch.monitor.os;
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Objects;
 
-/**
- *
- */
-public class OsStats implements Streamable, ToXContent {
-
-    long timestamp;
+public class OsStats implements Writeable, ToXContent {
 
-    Cpu cpu = null;
+    private final long timestamp;
+    private final Cpu cpu;
+    private final Mem mem;
+    private final Swap swap;
 
-    Mem mem = null;
+    public OsStats(long timestamp, Cpu cpu, Mem mem, Swap swap) {
+        this.timestamp = timestamp;
+        this.cpu = Objects.requireNonNull(cpu, "cpu must not be null");
+        this.mem = Objects.requireNonNull(mem, "mem must not be null");;
+        this.swap = Objects.requireNonNull(swap, "swap must not be null");;
+    }
 
-    Swap swap = null;
+    public OsStats(StreamInput in) throws IOException {
+        this.timestamp = in.readVLong();
+        this.cpu = new Cpu(in);
+        this.mem = new Mem(in);
+        this.swap = new Swap(in);
+    }
 
-    OsStats() {
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVLong(timestamp);
+        cpu.writeTo(out);
+        mem.writeTo(out);
+        swap.writeTo(out);
     }
 
     public long getTimestamp() {
@@ -65,9 +79,9 @@ public class OsStats implements Streamable, ToXContent {
         static final String CPU = "cpu";
         static final String PERCENT = "percent";
         static final String LOAD_AVERAGE = "load_average";
-        static final String LOAD_AVERAGE_1M = new String("1m");
-        static final String LOAD_AVERAGE_5M = new String("5m");
-        static final String LOAD_AVERAGE_15M = new String("15m");
+        static final String LOAD_AVERAGE_1M = "1m";
+        static final String LOAD_AVERAGE_5M = "5m";
+        static final String LOAD_AVERAGE_15M = "15m";
 
         static final String MEM = "mem";
         static final String SWAP = "swap";
@@ -86,105 +100,29 @@ public class OsStats implements Streamable, ToXContent {
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(Fields.OS);
         builder.field(Fields.TIMESTAMP, getTimestamp());
-        if (cpu != null) {
-            builder.startObject(Fields.CPU);
-            builder.field(Fields.PERCENT, cpu.getPercent());
-            if (cpu.getLoadAverage() != null && Arrays.stream(cpu.getLoadAverage()).anyMatch(load -> load != -1)) {
-                builder.startObject(Fields.LOAD_AVERAGE);
-                if (cpu.getLoadAverage()[0] != -1) {
-                    builder.field(Fields.LOAD_AVERAGE_1M, cpu.getLoadAverage()[0]);
-                }
-                if (cpu.getLoadAverage()[1] != -1) {
-                    builder.field(Fields.LOAD_AVERAGE_5M, cpu.getLoadAverage()[1]);
-                }
-                if (cpu.getLoadAverage()[2] != -1) {
-                    builder.field(Fields.LOAD_AVERAGE_15M, cpu.getLoadAverage()[2]);
-                }
-                builder.endObject();
-            }
-            builder.endObject();
-        }
-
-        if (mem != null) {
-            builder.startObject(Fields.MEM);
-            builder.byteSizeField(Fields.TOTAL_IN_BYTES, Fields.TOTAL, mem.getTotal());
-            builder.byteSizeField(Fields.FREE_IN_BYTES, Fields.FREE, mem.getFree());
-            builder.byteSizeField(Fields.USED_IN_BYTES, Fields.USED, mem.getUsed());
-
-            builder.field(Fields.FREE_PERCENT, mem.getFreePercent());
-            builder.field(Fields.USED_PERCENT, mem.getUsedPercent());
-
-            builder.endObject();
-        }
-
-        if (swap != null) {
-            builder.startObject(Fields.SWAP);
-            builder.byteSizeField(Fields.TOTAL_IN_BYTES, Fields.TOTAL, swap.getTotal());
-            builder.byteSizeField(Fields.FREE_IN_BYTES, Fields.FREE, swap.getFree());
-            builder.byteSizeField(Fields.USED_IN_BYTES, Fields.USED, swap.getUsed());
-            builder.endObject();
-        }
-
+        cpu.toXContent(builder, params);
+        mem.toXContent(builder, params);
+        swap.toXContent(builder, params);
         builder.endObject();
         return builder;
     }
 
-    public static OsStats readOsStats(StreamInput in) throws IOException {
-        OsStats stats = new OsStats();
-        stats.readFrom(in);
-        return stats;
-    }
+    public static class Cpu implements Writeable, ToXContent {
 
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        timestamp = in.readVLong();
-        cpu = in.readOptionalStreamable(Cpu::new);
-        if (in.readBoolean()) {
-            mem = Mem.readMem(in);
-        }
-        if (in.readBoolean()) {
-            swap = Swap.readSwap(in);
-        }
-    }
+        private final short percent;
+        private final double[] loadAverage;
 
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeVLong(timestamp);
-        out.writeOptionalStreamable(cpu);
-        if (mem == null) {
-            out.writeBoolean(false);
-        } else {
-            out.writeBoolean(true);
-            mem.writeTo(out);
+        public Cpu(short systemCpuPercent, double[] systemLoadAverage) {
+            this.percent = systemCpuPercent;
+            this.loadAverage = systemLoadAverage;
         }
-        if (swap == null) {
-            out.writeBoolean(false);
-        } else {
-            out.writeBoolean(true);
-            swap.writeTo(out);
-        }
-    }
-
-    public static class Cpu implements Streamable {
-
-        short percent = -1;
-        double[] loadAverage = null;
-
-        Cpu() {}
 
-        public static Cpu readCpu(StreamInput in) throws IOException {
-            Cpu cpu = new Cpu();
-            cpu.readFrom(in);
-            return cpu;
-        }
-
-        @Override
-        public void readFrom(StreamInput in) throws IOException {
-            percent = in.readShort();
+        public Cpu(StreamInput in) throws IOException {
+            this.percent = in.readShort();
             if (in.readBoolean()) {
-                loadAverage = in.readDoubleArray();
+                this.loadAverage = in.readDoubleArray();
             } else {
-                loadAverage = null;
+                this.loadAverage = null;
             }
         }
 
@@ -206,12 +144,49 @@ public class OsStats implements Streamable, ToXContent {
         public double[] getLoadAverage() {
             return loadAverage;
         }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject(Fields.CPU);
+            builder.field(Fields.PERCENT, getPercent());
+            if (getLoadAverage() != null && Arrays.stream(getLoadAverage()).anyMatch(load -> load != -1)) {
+                builder.startObject(Fields.LOAD_AVERAGE);
+                if (getLoadAverage()[0] != -1) {
+                    builder.field(Fields.LOAD_AVERAGE_1M, getLoadAverage()[0]);
+                }
+                if (getLoadAverage()[1] != -1) {
+                    builder.field(Fields.LOAD_AVERAGE_5M, getLoadAverage()[1]);
+                }
+                if (getLoadAverage()[2] != -1) {
+                    builder.field(Fields.LOAD_AVERAGE_15M, getLoadAverage()[2]);
+                }
+                builder.endObject();
+            }
+            builder.endObject();
+            return builder;
+        }
     }
 
-    public static class Swap implements Streamable {
+    public static class Swap implements Writeable, ToXContent {
+
+        private final long total;
+        private final long free;
 
-        long total = -1;
-        long free = -1;
+        public Swap(long total, long free) {
+            this.total = total;
+            this.free = free;
+        }
+
+        public Swap(StreamInput in) throws IOException {
+            this.total = in.readLong();
+            this.free = in.readLong();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeLong(total);
+            out.writeLong(free);
+        }
 
         public ByteSizeValue getFree() {
             return new ByteSizeValue(free);
@@ -225,40 +200,30 @@ public class OsStats implements Streamable, ToXContent {
             return new ByteSizeValue(total);
         }
 
-        public static Swap readSwap(StreamInput in) throws IOException {
-            Swap swap = new Swap();
-            swap.readFrom(in);
-            return swap;
-        }
-
         @Override
-        public void readFrom(StreamInput in) throws IOException {
-            total = in.readLong();
-            free = in.readLong();
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeLong(total);
-            out.writeLong(free);
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject(Fields.SWAP);
+            builder.byteSizeField(Fields.TOTAL_IN_BYTES, Fields.TOTAL, getTotal());
+            builder.byteSizeField(Fields.FREE_IN_BYTES, Fields.FREE, getFree());
+            builder.byteSizeField(Fields.USED_IN_BYTES, Fields.USED, getUsed());
+            builder.endObject();
+            return builder;
         }
     }
 
-    public static class Mem implements Streamable {
+    public static class Mem implements Writeable, ToXContent {
 
-        long total = -1;
-        long free = -1;
+        private final long total;
+        private final long free;
 
-        public static Mem readMem(StreamInput in) throws IOException {
-            Mem mem = new Mem();
-            mem.readFrom(in);
-            return mem;
+        public Mem(long total, long free) {
+            this.total = total;
+            this.free = free;
         }
 
-        @Override
-        public void readFrom(StreamInput in) throws IOException {
-            total = in.readLong();
-            free = in.readLong();
+        public Mem(StreamInput in) throws IOException {
+            this.total = in.readLong();
+            this.free = in.readLong();
         }
 
         @Override
@@ -276,7 +241,7 @@ public class OsStats implements Streamable, ToXContent {
         }
 
         public short getUsedPercent() {
-            return calculatePercentage(getUsed().bytes(), getTotal().bytes());
+            return calculatePercentage(getUsed().bytes(), total);
         }
 
         public ByteSizeValue getFree() {
@@ -284,11 +249,23 @@ public class OsStats implements Streamable, ToXContent {
         }
 
         public short getFreePercent() {
-            return calculatePercentage(getFree().bytes(), getTotal().bytes());
+            return calculatePercentage(free, total);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject(Fields.MEM);
+            builder.byteSizeField(Fields.TOTAL_IN_BYTES, Fields.TOTAL, getTotal());
+            builder.byteSizeField(Fields.FREE_IN_BYTES, Fields.FREE, getFree());
+            builder.byteSizeField(Fields.USED_IN_BYTES, Fields.USED, getUsed());
+            builder.field(Fields.FREE_PERCENT, getFreePercent());
+            builder.field(Fields.USED_PERCENT, getUsedPercent());
+            builder.endObject();
+            return builder;
         }
     }
 
-    private static short calculatePercentage(long used, long max) {
+    public static short calculatePercentage(long used, long max) {
         return max <= 0 ? 0 : (short) (Math.round((100d * used) / max));
     }
 }

core/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java

@@ -21,12 +21,15 @@ package org.elasticsearch.action.admin.cluster.stats;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
+import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
+import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.monitor.os.OsStats;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
@@ -181,6 +184,20 @@ public class ClusterStatsIT extends ESIntegTestCase {
         assertThat(msg, response.nodesStats.getProcess().getMinOpenFileDescriptors(), Matchers.greaterThanOrEqualTo(-1L));
         assertThat(msg, response.nodesStats.getProcess().getMaxOpenFileDescriptors(), Matchers.greaterThanOrEqualTo(-1L));
 
+        NodesStatsResponse nodesStatsResponse = client().admin().cluster().prepareNodesStats().setOs(true).get();
+        long total = 0;
+        long free = 0;
+        long used = 0;
+        for (NodeStats nodeStats : nodesStatsResponse.getNodes()) {
+            total += nodeStats.getOs().getMem().getTotal().bytes();
+            free += nodeStats.getOs().getMem().getFree().bytes();
+            used += nodeStats.getOs().getMem().getUsed().bytes();
+        }
+        assertEquals(msg, free, response.nodesStats.getOs().getMem().getFree().bytes());
+        assertEquals(msg, total, response.nodesStats.getOs().getMem().getTotal().bytes());
+        assertEquals(msg, used, response.nodesStats.getOs().getMem().getUsed().bytes());
+        assertEquals(msg, OsStats.calculatePercentage(used, total), response.nodesStats.getOs().getMem().getUsedPercent());
+        assertEquals(msg, OsStats.calculatePercentage(free, total), response.nodesStats.getOs().getMem().getFreePercent());
     }
 
     public void testAllocatedProcessors() throws Exception {

core/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java

@@ -49,7 +49,7 @@ public class OsProbeTests extends ESTestCase {
         assertNotNull(stats);
         assertThat(stats.getTimestamp(), greaterThan(0L));
         assertThat(stats.getCpu().getPercent(), anyOf(equalTo((short) -1), is(both(greaterThanOrEqualTo((short) 0)).and(lessThanOrEqualTo((short) 100)))));
-        double[] loadAverage = stats.getCpu().loadAverage;
+        double[] loadAverage = stats.getCpu().getLoadAverage();
         if (loadAverage != null) {
             assertThat(loadAverage.length, equalTo(3));
         }

core/src/test/java/org/elasticsearch/monitor/os/OsStatsTests.java

@@ -0,0 +1,55 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.monitor.os;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+public class OsStatsTests extends ESTestCase {
+
+    public void testSerialization() throws IOException {
+        int numLoadAverages = randomIntBetween(1, 5);
+        double loadAverages[] = new double[numLoadAverages];
+        for (int i = 0; i < loadAverages.length; i++) {
+            loadAverages[i] = randomDouble();
+        }
+        OsStats.Cpu cpu = new OsStats.Cpu(randomShort(), loadAverages);
+        OsStats.Mem mem = new OsStats.Mem(randomLong(), randomLong());
+        OsStats.Swap swap = new OsStats.Swap(randomLong(), randomLong());
+        OsStats osStats = new OsStats(System.currentTimeMillis(), cpu, mem, swap);
+
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            osStats.writeTo(out);
+            try (StreamInput in = out.bytes().streamInput()) {
+                OsStats deserializedOsStats = new OsStats(in);
+                assertEquals(osStats.getTimestamp(), deserializedOsStats.getTimestamp());
+                assertEquals(osStats.getCpu().getPercent(), deserializedOsStats.getCpu().getPercent());
+                assertArrayEquals(osStats.getCpu().getLoadAverage(), deserializedOsStats.getCpu().getLoadAverage(), 0);
+                assertEquals(osStats.getMem().getFree(), deserializedOsStats.getMem().getFree());
+                assertEquals(osStats.getMem().getTotal(), deserializedOsStats.getMem().getTotal());
+                assertEquals(osStats.getSwap().getFree(), deserializedOsStats.getSwap().getFree());
+                assertEquals(osStats.getSwap().getTotal(), deserializedOsStats.getSwap().getTotal());
+            }
+        }
+    }
+}

docs/reference/cluster/stats.asciidoc

@@ -116,7 +116,17 @@ Will return, for example:
                "name": "Mac OS X",
                "count": 1
             }
-         ]
+         ],
+         "mem" : {
+            "total" : "16gb",
+            "total_in_bytes" : 17179869184,
+            "free" : "78.1mb",
+            "free_in_bytes" : 81960960,
+            "used" : "15.9gb",
+            "used_in_bytes" : 17097908224,
+            "free_percent" : 0,
+            "used_percent" : 100
+         }
       },
       "process": {
          "cpu": {

docs/reference/migration/migrate_5_0/java.asciidoc

@@ -349,7 +349,9 @@ The `setQuery(BytesReference)` method have been removed in favor of using `setQu
 ==== ClusterStatsResponse
 
 Removed the `getMemoryAvailable` method from `OsStats`, which could be previously accessed calling
-`clusterStatsResponse.getNodesStats().getOs().getMemoryAvailable()`.
+`clusterStatsResponse.getNodesStats().getOs().getMemoryAvailable()`. It is now replaced with
+`clusterStatsResponse.getNodesStats().getOs().getMem()` which exposes `getTotal()`, `getFree()`,
+`getUsed()`, `getFreePercent()` and `getUsedPercent()`.
 
 ==== setRefresh(boolean) has been removed
 

docs/reference/migration/migrate_5_0/rest.asciidoc

@@ -29,9 +29,9 @@ document exists in an index. The old endpoint will keep working until 6.0.
 
 ==== Removed `mem` section from `/_cluster/stats` response
 
-The `mem` section contained only one value, the total memory available
-throughout all nodes in the cluster. The section was removed as it didn't
-prove useful.
+The `mem` section contained only the `total` value, which was actually the
+memory available throughout all nodes in the cluster. The section contains now
+`total`, `free`, `used`, `used_percent` and `free_percent`.
 
 ==== Revised node roles aggregate returned by `/_cluster/stats`
 

rest-api-spec/src/main/resources/rest-api-spec/test/cluster.stats/10_basic.yaml

@@ -19,6 +19,11 @@
   - gte: { nodes.count.ingest: 0}
   - gte: { nodes.count.coordinating_only: 0}
   - is_true: nodes.os
+  - is_true: nodes.os.mem.total_in_bytes
+  - is_true: nodes.os.mem.free_in_bytes
+  - is_true: nodes.os.mem.used_in_bytes
+  - is_true: nodes.os.mem.free_percent
+  - is_true: nodes.os.mem.used_percent
   - is_true: nodes.process
   - is_true: nodes.jvm
   - is_true: nodes.fs