add mem section back to cluster stats

The mem section was buggy in cluster stats and removed. It is now added back with the same structure as in node stats, containing total memory, available memory, used memory and percentages. All the values are the sum of all the nodes across the cluster (or at least the ones that we were able to get the values from).

core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java

@@ -87,7 +87,7 @@ public class ClusterStatsNodes implements ToXContent {
             }
         }
         this.counts = new Counts(nodeInfos);
-        this.os = new OsStats(nodeInfos);
+        this.os = new OsStats(nodeInfos, nodeStats);
         this.process = new ProcessStats(nodeStats);
         this.jvm = new JvmStats(nodeInfos, nodeStats);
         this.networkTypes = new NetworkTypes(nodeInfos);
@@ -226,11 +226,12 @@ public class ClusterStatsNodes implements ToXContent {
         final int availableProcessors;
         final int allocatedProcessors;
         final ObjectIntHashMap<String> names;
+        final org.elasticsearch.monitor.os.OsStats.Mem mem;
 
         /**
          * Build the stats from information about each node.
          */
-        private OsStats(List<NodeInfo> nodeInfos) {
+        private OsStats(List<NodeInfo> nodeInfos, List<NodeStats> nodeStatsList) {
             this.names = new ObjectIntHashMap<>();
             int availableProcessors = 0;
             int allocatedProcessors = 0;
@@ -244,6 +245,22 @@ public class ClusterStatsNodes implements ToXContent {
             }
             this.availableProcessors = availableProcessors;
             this.allocatedProcessors = allocatedProcessors;
+
+            long totalMemory = 0;
+            long freeMemory = 0;
+            for (NodeStats nodeStats : nodeStatsList) {
+                if (nodeStats.getOs() != null) {
+                    long total = nodeStats.getOs().getMem().getTotal().bytes();
+                    if (total > 0) {
+                        totalMemory += total;
+                    }
+                    long free = nodeStats.getOs().getMem().getFree().bytes();
+                    if (free > 0) {
+                        freeMemory += free;
+                    }
+                }
+            }
+            this.mem = new org.elasticsearch.monitor.os.OsStats.Mem(totalMemory, freeMemory);
         }
 
         public int getAvailableProcessors() {
@@ -254,6 +271,10 @@ public class ClusterStatsNodes implements ToXContent {
             return allocatedProcessors;
         }
 
+        public org.elasticsearch.monitor.os.OsStats.Mem getMem() {
+            return mem;
+        }
+
         static final class Fields {
             static final String AVAILABLE_PROCESSORS = "available_processors";
             static final String ALLOCATED_PROCESSORS = "allocated_processors";
@@ -274,6 +295,7 @@ public class ClusterStatsNodes implements ToXContent {
                 builder.endObject();
             }
             builder.endArray();
+            mem.toXContent(builder, params);
             return builder;
         }
     }

core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java

@@ -92,7 +92,7 @@ public class TransportClusterStatsAction extends TransportNodesAction<ClusterSta
     @Override
     protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeRequest) {
         NodeInfo nodeInfo = nodeService.info(true, true, false, true, false, true, false, true, false, false);
-        NodeStats nodeStats = nodeService.stats(CommonStatsFlags.NONE, false, true, true, false, true, false, false, false, false, false, false);
+        NodeStats nodeStats = nodeService.stats(CommonStatsFlags.NONE, true, true, true, false, true, false, false, false, false, false, false);
         List<ShardStats> shardsStats = new ArrayList<>();
         for (IndexService indexService : indicesService) {
             for (IndexShard indexShard : indexService) {

core/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java

@@ -181,6 +181,12 @@ public class ClusterStatsIT extends ESIntegTestCase {
         assertThat(msg, response.nodesStats.getProcess().getMinOpenFileDescriptors(), Matchers.greaterThanOrEqualTo(-1L));
         assertThat(msg, response.nodesStats.getProcess().getMaxOpenFileDescriptors(), Matchers.greaterThanOrEqualTo(-1L));
 
+        assertThat(msg, response.nodesStats.getOs().getMem().getFree().bytes(), Matchers.greaterThanOrEqualTo(0L));
+        assertThat(msg, response.nodesStats.getOs().getMem().getTotal().bytes(), Matchers.greaterThanOrEqualTo(0L));
+        assertThat(msg, response.nodesStats.getOs().getMem().getUsed().bytes(), Matchers.greaterThanOrEqualTo(0L));
+        assertThat(msg, response.nodesStats.getOs().getMem().getUsedPercent(), Matchers.greaterThanOrEqualTo((short)0));
+        assertThat(msg, response.nodesStats.getOs().getMem().getFreePercent(), Matchers.greaterThanOrEqualTo((short)0));
+
     }
 
     public void testAllocatedProcessors() throws Exception {

docs/reference/cluster/stats.asciidoc

@@ -116,7 +116,17 @@ Will return, for example:
                "name": "Mac OS X",
                "count": 1
             }
-         ]
+         ],
+         "mem" : {
+            "total" : "16gb",
+            "total_in_bytes" : 17179869184,
+            "free" : "78.1mb",
+            "free_in_bytes" : 81960960,
+            "used" : "15.9gb",
+            "used_in_bytes" : 17097908224,
+            "free_percent" : 0,
+            "used_percent" : 100
+         }
       },
       "process": {
          "cpu": {

docs/reference/migration/migrate_5_0/java.asciidoc

@@ -349,7 +349,9 @@ The `setQuery(BytesReference)` method have been removed in favor of using `setQu
 ==== ClusterStatsResponse
 
 Removed the `getMemoryAvailable` method from `OsStats`, which could be previously accessed calling
-`clusterStatsResponse.getNodesStats().getOs().getMemoryAvailable()`.
+`clusterStatsResponse.getNodesStats().getOs().getMemoryAvailable()`. It is now replaced with
+`clusterStatsResponse.getNodesStats().getOs().getMem()` which exposes `getTotal()`, `getFree()`,
+`getUsed()`, `getFreePercent()` and `getUsedPercent()`.
 
 ==== setRefresh(boolean) has been removed
 

docs/reference/migration/migrate_5_0/rest.asciidoc

@@ -29,9 +29,9 @@ document exists in an index. The old endpoint will keep working until 6.0.
 
 ==== Removed `mem` section from `/_cluster/stats` response
 
-The `mem` section contained only one value, the total memory available
-throughout all nodes in the cluster. The section was removed as it didn't
-prove useful.
+The `mem` section contained only the `total` value, which was actually the
+memory available throughout all nodes in the cluster. The section contains now
+`total`, `free`, `used`, `used_percent` and `free_percent`.
 
 ==== Revised node roles aggregate returned by `/_cluster/stats`
 

rest-api-spec/src/main/resources/rest-api-spec/test/cluster.stats/10_basic.yaml

@@ -19,6 +19,11 @@
   - gte: { nodes.count.ingest: 0}
   - gte: { nodes.count.coordinating_only: 0}
   - is_true: nodes.os
+  - is_true: nodes.os.mem.total_in_bytes
+  - is_true: nodes.os.mem.free_in_bytes
+  - is_true: nodes.os.mem.used_in_bytes
+  - is_true: nodes.os.mem.free_percent
+  - is_true: nodes.os.mem.used_percent
   - is_true: nodes.process
   - is_true: nodes.jvm
   - is_true: nodes.fs