Add additional logging to make spotting stats issues easier (#133972)

* Add additional logging to make spotting stats issues easier

* Refactor logging in JVM monitoring classes for improved error handling

* Switch to debug level logging

server/src/main/java/org/elasticsearch/monitor/Probes.java

@@ -9,11 +9,17 @@
 
 package org.elasticsearch.monitor;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
 import java.lang.management.OperatingSystemMXBean;
 import java.lang.reflect.Method;
 
 public class Probes {
+    private static final Logger logger = LogManager.getLogger(Probes.class);
+
     public static short getLoadAndScaleToPercent(Method method, OperatingSystemMXBean osMxBean) {
+        logger.debug("Starting probe of method {} on osMxBean {}", method, osMxBean);
         if (method != null) {
             try {
                 double load = (double) method.invoke(osMxBean);
@@ -21,9 +27,11 @@ public class Probes {
                     return (short) (load * 100);
                 }
             } catch (Exception e) {
+                logger.debug(() -> "failed to invoke method [" + method + "] on osMxBean [" + osMxBean + "]", e);
                 return -1;
             }
         }
+        logger.debug("Method is null. Returning default value.");
         return -1;
     }
 }

server/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.monitor.jvm;
 
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -30,10 +31,14 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
+import static org.apache.logging.log4j.LogManager.getLogger;
+
 public class JvmInfo implements ReportingService.Info {
 
     private static final JvmInfo INSTANCE;
 
+    private static final Logger logger = getLogger(JvmInfo.class);
+
     static {
         RuntimeMXBean runtimeMXBean = ManagementFactory.getRuntimeMXBean();
         MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean();
@@ -94,47 +99,63 @@ public class JvmInfo implements ReportingService.Info {
             try {
                 Object onErrorObject = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "OnError");
                 onError = (String) valueMethod.invoke(onErrorObject);
-            } catch (Exception ignored) {}
+            } catch (Exception e) {
+                logger.debug("Error getting JVM info from MX Bean", e);
+            }
 
             try {
                 Object onOutOfMemoryErrorObject = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "OnOutOfMemoryError");
                 onOutOfMemoryError = (String) valueMethod.invoke(onOutOfMemoryErrorObject);
-            } catch (Exception ignored) {}
+            } catch (Exception e) {
+                logger.debug("Error getting JVM info from MX Bean", e);
+            }
 
             try {
                 Object useCompressedOopsVmOptionObject = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "UseCompressedOops");
                 useCompressedOops = (String) valueMethod.invoke(useCompressedOopsVmOptionObject);
-            } catch (Exception ignored) {}
+            } catch (Exception e) {
+                logger.debug("Error getting JVM info from MX Bean", e);
+            }
 
             try {
                 Object useG1GCVmOptionObject = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "UseG1GC");
                 useG1GC = (String) valueMethod.invoke(useG1GCVmOptionObject);
                 Object regionSizeVmOptionObject = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "G1HeapRegionSize");
                 g1RegisionSize = Long.parseLong((String) valueMethod.invoke(regionSizeVmOptionObject));
-            } catch (Exception ignored) {}
+            } catch (Exception e) {
+                logger.debug("Error getting JVM info from MX Bean", e);
+            }
 
             try {
                 Object initialHeapSizeVmOptionObject = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "InitialHeapSize");
                 configuredInitialHeapSize = Long.parseLong((String) valueMethod.invoke(initialHeapSizeVmOptionObject));
-            } catch (Exception ignored) {}
+            } catch (Exception e) {
+                logger.debug("Error getting JVM info from MX Bean", e);
+            }
 
             try {
                 Object maxHeapSizeVmOptionObject = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "MaxHeapSize");
                 configuredMaxHeapSize = Long.parseLong((String) valueMethod.invoke(maxHeapSizeVmOptionObject));
-            } catch (Exception ignored) {}
+            } catch (Exception e) {
+                logger.debug("Error getting JVM info from MX Bean", e);
+            }
 
             try {
                 Object maxDirectMemorySizeVmOptionObject = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "MaxDirectMemorySize");
                 directMemoryMax = Long.parseLong((String) valueMethod.invoke(maxDirectMemorySizeVmOptionObject));
-            } catch (Exception ignored) {}
+            } catch (Exception e) {
+                logger.debug("Error getting JVM info from MX Bean", e);
+            }
 
             try {
                 Object useSerialGCVmOptionObject = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "UseSerialGC");
                 useSerialGC = (String) valueMethod.invoke(useSerialGCVmOptionObject);
-            } catch (Exception ignored) {}
-
-        } catch (Exception ignored) {
+            } catch (Exception e) {
+                logger.debug("Error getting JVM info from MX Bean", e);
+            }
 
+        } catch (Exception e) {
+            logger.debug("Error getting JVM info from MX Bean", e);
         }
 
         Mem mem = new Mem(heapInit, heapMax, nonHeapInit, nonHeapMax, directMemoryMax);

server/src/main/java/org/elasticsearch/monitor/jvm/SunThreadInfo.java

@@ -9,13 +9,14 @@
 
 package org.elasticsearch.monitor.jvm;
 
-import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
 import java.lang.management.ManagementFactory;
 import java.lang.management.ThreadMXBean;
 import java.lang.reflect.Method;
 
+import static org.apache.logging.log4j.LogManager.getLogger;
+
 public class SunThreadInfo {
 
     private static final ThreadMXBean threadMXBean;
@@ -23,7 +24,7 @@ public class SunThreadInfo {
     private static final Method isThreadAllocatedMemorySupported;
     private static final Method isThreadAllocatedMemoryEnabled;
 
-    private static final Logger logger = LogManager.getLogger(SunThreadInfo.class);
+    private static final Logger logger = getLogger(SunThreadInfo.class);
     public static final SunThreadInfo INSTANCE = new SunThreadInfo();
 
     static {
@@ -83,11 +84,13 @@ public class SunThreadInfo {
     }
 
     private static Method getMethod(String methodName, Class<?>... parameterTypes) {
+        String className = "com.sun.management.ThreadMXBean";
         try {
-            Method method = Class.forName("com.sun.management.ThreadMXBean").getMethod(methodName, parameterTypes);
+            Method method = Class.forName(className).getMethod(methodName, parameterTypes);
             return method;
         } catch (Exception e) {
             // not available
+            logger.debug(() -> "failed to get method [" + methodName + "] from class [" + className + "]", e);
             return null;
         }
     }

server/src/main/java/org/elasticsearch/monitor/os/OsProbe.java

@@ -773,7 +773,7 @@ public class OsProbe {
 
     }
 
-    private final Logger logger = LogManager.getLogger(getClass());
+    private static final Logger logger = LogManager.getLogger(OsProbe.class);
 
     OsInfo osInfo(long refreshInterval, Processors processors) throws IOException {
         return new OsInfo(
@@ -913,10 +913,12 @@ public class OsProbe {
      * Returns a given method of the OperatingSystemMXBean, or null if the method is not found or unavailable.
      */
     private static Method getMethod(String methodName) {
+        String className = "com.sun.management.OperatingSystemMXBean";
         try {
-            return Class.forName("com.sun.management.OperatingSystemMXBean").getMethod(methodName);
+            return Class.forName(className).getMethod(methodName);
         } catch (Exception e) {
             // not available
+            logger.debug(() -> "failed to get method [" + methodName + "] from class [" + className + "]", e);
             return null;
         }
     }

server/src/main/java/org/elasticsearch/monitor/process/ProcessProbe.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.monitor.process;
 
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.bootstrap.BootstrapInfo;
 import org.elasticsearch.monitor.Probes;
 
@@ -16,10 +17,13 @@ import java.lang.management.ManagementFactory;
 import java.lang.management.OperatingSystemMXBean;
 import java.lang.reflect.Method;
 
+import static org.apache.logging.log4j.LogManager.getLogger;
 import static org.elasticsearch.monitor.jvm.JvmInfo.jvmInfo;
 
 public class ProcessProbe {
 
+    private static final Logger logger = getLogger(ProcessProbe.class);
+
     private static final OperatingSystemMXBean osMxBean = ManagementFactory.getOperatingSystemMXBean();
 
     private static final Method getMaxFileDescriptorCountField;
@@ -130,10 +134,12 @@ public class ProcessProbe {
      * or null if the method is not found or unavailable.
      */
     private static Method getMethod(String methodName) {
+        String className = "com.sun.management.OperatingSystemMXBean";
         try {
-            return Class.forName("com.sun.management.OperatingSystemMXBean").getMethod(methodName);
-        } catch (Exception t) {
+            return Class.forName(className).getMethod(methodName);
+        } catch (Exception e) {
             // not available
+            logger.debug(() -> "failed to get method [" + methodName + "] from class [" + className + "]", e);
             return null;
         }
     }
@@ -143,10 +149,12 @@ public class ProcessProbe {
      * or null if the method is not found or unavailable.
      */
     private static Method getUnixMethod(String methodName) {
+        String className = "com.sun.management.UnixOperatingSystemMXBean";
         try {
-            return Class.forName("com.sun.management.UnixOperatingSystemMXBean").getMethod(methodName);
-        } catch (Exception t) {
+            return Class.forName(className).getMethod(methodName);
+        } catch (Exception e) {
             // not available
+            logger.debug(() -> "failed to get method [" + methodName + "] from class [" + className + "]", e);
             return null;
         }
     }