Merge pull request #14741 from jasontedor/return-of-the-cpu-percent

Add system CPU percent to OS stats

core/src/main/java/org/elasticsearch/monitor/Probes.java

@@ -0,0 +1,20 @@
+package org.elasticsearch.monitor;
+
+import java.lang.management.OperatingSystemMXBean;
+import java.lang.reflect.Method;
+
+public class Probes {
+    public static short getLoadAndScaleToPercent(Method method, OperatingSystemMXBean osMxBean) {
+        if (method != null) {
+            try {
+                double load = (double) method.invoke(osMxBean);
+                if (load >= 0) {
+                    return (short) (load * 100);
+                }
+            } catch (Throwable t) {
+                return -1;
+            }
+        }
+        return -1;
+    }
+}

core/src/main/java/org/elasticsearch/monitor/os/OsProbe.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.monitor.os;
 
 import org.apache.lucene.util.Constants;
+import org.elasticsearch.monitor.Probes;
 
 import java.lang.management.ManagementFactory;
 import java.lang.management.OperatingSystemMXBean;
@@ -34,6 +35,7 @@ public class OsProbe {
     private static final Method getFreeSwapSpaceSize;
     private static final Method getTotalSwapSpaceSize;
     private static final Method getSystemLoadAverage;
+    private static final Method getSystemCpuLoad;
 
     static {
         getFreePhysicalMemorySize = getMethod("getFreePhysicalMemorySize");
@@ -41,6 +43,7 @@ public class OsProbe {
         getFreeSwapSpaceSize = getMethod("getFreeSwapSpaceSize");
         getTotalSwapSpaceSize = getMethod("getTotalSwapSpaceSize");
         getSystemLoadAverage = getMethod("getSystemLoadAverage");
+        getSystemCpuLoad = getMethod("getSystemCpuLoad");
     }
 
     /**
@@ -113,6 +116,10 @@ public class OsProbe {
         }
     }
 
+    public short getSystemCpuPercent() {
+        return Probes.getLoadAndScaleToPercent(getSystemCpuLoad, osMxBean);
+    }
+
     private static class OsProbeHolder {
         private final static OsProbe INSTANCE = new OsProbe();
     }
@@ -136,7 +143,9 @@ public class OsProbe {
     public OsStats osStats() {
         OsStats stats = new OsStats();
         stats.timestamp = System.currentTimeMillis();
-        stats.loadAverage = getSystemLoadAverage();
+        stats.cpu = new OsStats.Cpu();
+        stats.cpu.percent = getSystemCpuPercent();
+        stats.cpu.loadAverage = getSystemLoadAverage();
 
         OsStats.Mem mem = new OsStats.Mem();
         mem.total = getTotalPhysicalMemorySize();

core/src/main/java/org/elasticsearch/monitor/os/OsStats.java

@@ -36,7 +36,7 @@ public class OsStats implements Streamable, ToXContent {
 
     long timestamp;
 
-    double loadAverage = -1;
+    Cpu cpu = null;
 
     Mem mem = null;
 
@@ -49,10 +49,7 @@ public class OsStats implements Streamable, ToXContent {
         return timestamp;
     }
 
-    public double getLoadAverage() {
-        return loadAverage;
-    }
-
+    public Cpu getCpu() { return cpu; }
 
     public Mem getMem() {
         return mem;
@@ -65,6 +62,8 @@ public class OsStats implements Streamable, ToXContent {
     static final class Fields {
         static final XContentBuilderString OS = new XContentBuilderString("os");
         static final XContentBuilderString TIMESTAMP = new XContentBuilderString("timestamp");
+        static final XContentBuilderString CPU = new XContentBuilderString("cpu");
+        static final XContentBuilderString PERCENT = new XContentBuilderString("percent");
         static final XContentBuilderString LOAD_AVERAGE = new XContentBuilderString("load_average");
 
         static final XContentBuilderString MEM = new XContentBuilderString("mem");
@@ -85,7 +84,12 @@ public class OsStats implements Streamable, ToXContent {
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(Fields.OS);
         builder.field(Fields.TIMESTAMP, getTimestamp());
-        builder.field(Fields.LOAD_AVERAGE, getLoadAverage());
+        if (cpu != null) {
+            builder.startObject(Fields.CPU);
+            builder.field(Fields.PERCENT, cpu.getPercent());
+            builder.field(Fields.LOAD_AVERAGE, cpu.getLoadAverage());
+            builder.endObject();
+        }
 
         if (mem != null) {
             builder.startObject(Fields.MEM);
@@ -120,7 +124,7 @@ public class OsStats implements Streamable, ToXContent {
     @Override
     public void readFrom(StreamInput in) throws IOException {
         timestamp = in.readVLong();
-        loadAverage = in.readDouble();
+        cpu = in.readOptionalStreamable(Cpu::new);
         if (in.readBoolean()) {
             mem = Mem.readMem(in);
         }
@@ -132,7 +136,7 @@ public class OsStats implements Streamable, ToXContent {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeVLong(timestamp);
-        out.writeDouble(loadAverage);
+        out.writeOptionalStreamable(cpu);
         if (mem == null) {
             out.writeBoolean(false);
         } else {
@@ -147,6 +151,39 @@ public class OsStats implements Streamable, ToXContent {
         }
     }
 
+    public static class Cpu implements Streamable {
+        short percent = -1;
+        double loadAverage = -1;
+
+        Cpu() {}
+
+        public static Cpu readCpu(StreamInput in) throws IOException {
+            Cpu cpu = new Cpu();
+            cpu.readFrom(in);
+            return cpu;
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            percent = in.readShort();
+            loadAverage = in.readDouble();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeShort(percent);
+            out.writeDouble(loadAverage);
+        }
+
+        public short getPercent() {
+            return percent;
+        }
+
+        public double getLoadAverage() {
+            return loadAverage;
+        }
+    }
+
     public static class Swap implements Streamable {
 
         long total = -1;
@@ -230,5 +267,4 @@ public class OsStats implements Streamable, ToXContent {
     private static short calculatePercentage(long used, long max) {
         return max <= 0 ? 0 : (short) (Math.round((100d * used) / max));
     }
-
 }

core/src/main/java/org/elasticsearch/monitor/process/ProcessProbe.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.monitor.process;
 
 import org.elasticsearch.bootstrap.BootstrapInfo;
+import org.elasticsearch.monitor.Probes;
 
 import java.lang.management.ManagementFactory;
 import java.lang.management.OperatingSystemMXBean;
@@ -88,17 +89,7 @@ public class ProcessProbe {
      * Returns the process CPU usage in percent
      */
     public short getProcessCpuPercent() {
-        if (getProcessCpuLoad != null) {
-            try {
-                double load = (double) getProcessCpuLoad.invoke(osMxBean);
-                if (load >= 0) {
-                    return (short) (load * 100);
-                }
-            } catch (Throwable t) {
-                return -1;
-            }
-        }
-        return -1;
+        return Probes.getLoadAndScaleToPercent(getProcessCpuLoad, osMxBean);
     }
 
     /**

core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java

@@ -130,6 +130,7 @@ public class RestNodesAction extends AbstractCatAction {
         table.addCell("file_desc.percent", "default:false;alias:fdp,fileDescriptorPercent;text-align:right;desc:used file descriptor ratio");
         table.addCell("file_desc.max", "default:false;alias:fdm,fileDescriptorMax;text-align:right;desc:max file descriptors");
 
+        table.addCell("cpu", "alias:cpu;text-align:right;desc:recent cpu usage");
         table.addCell("load", "alias:l;text-align:right;desc:most recent load avg");
         table.addCell("uptime", "default:false;alias:u;text-align:right;desc:node uptime");
         table.addCell("node.role", "alias:r,role,dc,nodeRole;desc:d:data node, c:client node");
@@ -258,7 +259,8 @@ public class RestNodesAction extends AbstractCatAction {
             table.addCell(processStats == null ? null : calculatePercentage(processStats.getOpenFileDescriptors(), processStats.getMaxFileDescriptors()));
             table.addCell(processStats == null ? null : processStats.getMaxFileDescriptors());
 
-            table.addCell(osStats == null ? null : String.format(Locale.ROOT, "%.2f", osStats.getLoadAverage()));
+            table.addCell(osStats == null ? null : Short.toString(osStats.getCpu().getPercent()));
+            table.addCell(osStats == null ? null : String.format(Locale.ROOT, "%.2f", osStats.getCpu().getLoadAverage()));
             table.addCell(jvmStats == null ? null : jvmStats.getUptime());
             table.addCell(node.clientNode() ? "c" : node.dataNode() ? "d" : "-");
             table.addCell(masterId == null ? "x" : masterId.equals(node.id()) ? "*" : node.masterNode() ? "m" : "-");

core/src/test/java/org/elasticsearch/benchmark/monitor/os/OsProbeBenchmark.java

@@ -41,12 +41,10 @@ public class OsProbeBenchmark {
             probe.getTotalSwapSpaceSize();
             probe.getFreeSwapSpaceSize();
             probe.getSystemLoadAverage();
+            probe.getSystemCpuPercent();
         }
         logger.info("--> warmed up");
 
-
-
-
         logger.info("--> testing 'getTotalPhysicalMemorySize' method...");
         long start = System.currentTimeMillis();
         for (int i = 0; i < ITERATIONS; i++) {
@@ -86,5 +84,13 @@ public class OsProbeBenchmark {
         }
         elapsed = System.currentTimeMillis() - start;
         logger.info("--> total [{}] ms, avg [{}] ms", elapsed, (elapsed / (double)ITERATIONS));
+
+        logger.info("--> testing 'getSystemCpuPercent' method...");
+        start = System.currentTimeMillis();
+        for (int i = 0; i < ITERATIONS; i++) {
+            probe.getSystemCpuPercent();
+        }
+        elapsed = System.currentTimeMillis() - start;
+        logger.info("--> total [{}] ms, avg [{}] ms", elapsed, (elapsed / (double)ITERATIONS));
     }
 }

core/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java

@@ -22,13 +22,7 @@ package org.elasticsearch.monitor.os;
 import org.apache.lucene.util.Constants;
 import org.elasticsearch.test.ESTestCase;
 
-import static org.hamcrest.Matchers.allOf;
-import static org.hamcrest.Matchers.anyOf;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThan;
-import static org.hamcrest.Matchers.greaterThanOrEqualTo;
-import static org.hamcrest.Matchers.lessThan;
-import static org.hamcrest.Matchers.lessThanOrEqualTo;
+import static org.hamcrest.Matchers.*;
 
 public class OsProbeTests extends ESTestCase {
     OsProbe probe = OsProbe.getInstance();
@@ -47,12 +41,13 @@ public class OsProbeTests extends ESTestCase {
         OsStats stats = probe.osStats();
         assertNotNull(stats);
         assertThat(stats.getTimestamp(), greaterThan(0L));
+        assertThat(stats.getCpu().getPercent(), anyOf(equalTo((short) -1), is(both(greaterThanOrEqualTo((short) 0)).and(lessThanOrEqualTo((short) 100)))));
         if (Constants.WINDOWS) {
             // Load average is always -1 on Windows platforms
-            assertThat(stats.getLoadAverage(), equalTo((double) -1));
+            assertThat(stats.getCpu().getLoadAverage(), equalTo((double) -1));
         } else {
             // Load average can be negative if not available or not computed yet, otherwise it should be >= 0
-            assertThat(stats.getLoadAverage(), anyOf(lessThan((double) 0), greaterThanOrEqualTo((double) 0)));
+            assertThat(stats.getCpu().getLoadAverage(), anyOf(lessThan((double) 0), greaterThanOrEqualTo((double) 0)));
         }
 
         assertNotNull(stats.getMem());

docs/reference/cluster/nodes-stats.asciidoc

@@ -128,7 +128,10 @@ the operating system:
 `os.timestamp`::
 	Last time the operating system statistics have been refreshed
 
-`os.load_average`::
+`os.cpu.percent`::
+    Recent CPU usage for the whole system, or -1 if not supported
+
+`os.cpu.load_average`::
 	System load average for the last minute, or -1 if not supported
 
 `os.mem.total_in_bytes`::

docs/reference/migration/migrate_3_0.asciidoc

@@ -428,3 +428,15 @@ controls the backing queue for the thread pool and modifying this is an expert s
 and high risk of being misused. The ability to change the thread pool type for any thread pool has been removed; do note
 that it is still possible to adjust relevant thread pool parameters for each of the thread pools (e.g., depending on
 the thread pool type, `keep_alive`, `queue_size`, etc.).
+
+=== Adding system CPU percent to OS stats
+
+The recent CPU usage (as a percent) has been added to the OS stats reported under the node stats API and the cat nodes
+API. The breaking change here is that there is a new object in the "os" object in the node stats response. This object
+is called "cpu" and includes "percent" and "load_average" as fields. This moves the "load_average" field that was
+previously a top-level field in the "os" object to the "cpu" object. Additionally, the "cpu" field in the cat nodes API
+response is output by default.
+
+Finally, the API for org.elasticsearch.monitor.os.OsStats has changed. The `getLoadAverage` method has been removed. The
+value for this can now be obtained from `OsStats.Cpu#getLoadAverage`. Additionally, the recent CPU usage can be obtained
+from `OsStats.Cpu#getPercent`.