Merge pull request #18756 from jasontedor/on-out-of-memory-error

Bootstrap check for OnOutOfMemoryError and seccomp

core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java

@@ -106,7 +106,7 @@ final class BootstrapCheck {
 
         for (final Check check : checks) {
             if (check.check()) {
-                if (!enforceLimits || (check.isSystemCheck() && ignoreSystemChecks)) {
+                if ((!enforceLimits || (check.isSystemCheck() && ignoreSystemChecks)) && !check.alwaysEnforce()) {
                     ignoredErrors.add(check.errorMessage());
                 } else {
                     errors.add(check.errorMessage());
@@ -164,6 +164,8 @@ final class BootstrapCheck {
             checks.add(new MaxMapCountCheck());
         }
         checks.add(new ClientJvmCheck());
+        checks.add(new OnErrorCheck());
+        checks.add(new OnOutOfMemoryErrorCheck());
         return Collections.unmodifiableList(checks);
     }
 
@@ -194,6 +196,10 @@ final class BootstrapCheck {
          */
         boolean isSystemCheck();
 
+        default boolean alwaysEnforce() {
+            return false;
+        }
+
     }
 
     static class HeapSizeCheck implements BootstrapCheck.Check {
@@ -245,7 +251,6 @@ final class BootstrapCheck {
 
     }
 
-    // visible for testing
     static class FileDescriptorCheck implements Check {
 
         private final int limit;
@@ -288,7 +293,6 @@ final class BootstrapCheck {
 
     }
 
-    // visible for testing
     static class MlockallCheck implements Check {
 
         private final boolean mlockallSet;
@@ -504,4 +508,81 @@ final class BootstrapCheck {
 
     }
 
+    static abstract class MightForkCheck implements BootstrapCheck.Check {
+
+        @Override
+        public boolean check() {
+            return isSeccompInstalled() && mightFork();
+        }
+
+        // visible for testing
+        boolean isSeccompInstalled() {
+            return Natives.isSeccompInstalled();
+        }
+
+        // visible for testing
+        abstract boolean mightFork();
+
+        @Override
+        public final boolean isSystemCheck() {
+            return false;
+        }
+
+        @Override
+        public final boolean alwaysEnforce() {
+            return true;
+        }
+
+    }
+
+    static class OnErrorCheck extends MightForkCheck {
+
+        @Override
+        boolean mightFork() {
+            final String onError = onError();
+            return onError != null && !onError.equals("");
+        }
+
+        // visible for testing
+        String onError() {
+            return JvmInfo.jvmInfo().onError();
+        }
+
+        @Override
+        public String errorMessage() {
+            return String.format(
+                Locale.ROOT,
+                "OnError [%s] requires forking but is prevented by system call filters ([%s=true]);" +
+                    " upgrade to at least Java 8u92 and use ExitOnOutOfMemoryError",
+                onError(),
+                BootstrapSettings.SECCOMP_SETTING.getKey());
+        }
+
+    }
+
+    static class OnOutOfMemoryErrorCheck extends MightForkCheck {
+
+        @Override
+        boolean mightFork() {
+            final String onOutOfMemoryError = onOutOfMemoryError();
+            return onOutOfMemoryError != null && !onOutOfMemoryError.equals("");
+        }
+
+        // visible for testing
+        String onOutOfMemoryError() {
+            return JvmInfo.jvmInfo().onOutOfMemoryError();
+        }
+
+        @Override
+        public String errorMessage() {
+            return String.format(
+                Locale.ROOT,
+                "OnOutOfMemoryError [%s] requires forking but is prevented by system call filters ([%s=true]);" +
+                    " upgrade to at least Java 8u92 and use ExitOnOutOfMemoryError",
+                onOutOfMemoryError(),
+                BootstrapSettings.SECCOMP_SETTING.getKey());
+        }
+
+    }
+
 }

core/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java

@@ -115,6 +115,18 @@ public class JvmInfo implements Streamable, ToXContent {
             Method vmOptionMethod = clazz.getMethod("getVMOption", String.class);
             Method valueMethod = vmOptionClazz.getMethod("getValue");
 
+            try {
+                Object onError = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "OnError");
+                info.onError = (String) valueMethod.invoke(onError);
+            } catch (Exception ignored) {
+            }
+
+            try {
+                Object onOutOfMemoryError = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "OnOutOfMemoryError");
+                info.onOutOfMemoryError = (String) valueMethod.invoke(onOutOfMemoryError);
+            } catch (Exception ignored) {
+            }
+
             try {
                 Object useCompressedOopsVmOption = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "UseCompressedOops");
                 info.useCompressedOops = (String) valueMethod.invoke(useCompressedOopsVmOption);
@@ -179,6 +191,10 @@ public class JvmInfo implements Streamable, ToXContent {
     String[] gcCollectors = Strings.EMPTY_ARRAY;
     String[] memoryPools = Strings.EMPTY_ARRAY;
 
+    private String onError;
+
+    private String onOutOfMemoryError;
+
     private String useCompressedOops = "unknown";
 
     private String useG1GC = "unknown";
@@ -314,6 +330,14 @@ public class JvmInfo implements Streamable, ToXContent {
         return configuredMaxHeapSize;
     }
 
+    public String onError() {
+        return onError;
+    }
+
+    public String onOutOfMemoryError() {
+        return onOutOfMemoryError;
+    }
+
     /**
      * The value of the JVM flag UseCompressedOops, if available otherwise
      * "unknown". The value "unknown" indicates that an attempt was

core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java

@@ -30,8 +30,10 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
 
 import static org.hamcrest.CoreMatchers.allOf;
 import static org.hamcrest.CoreMatchers.containsString;
@@ -383,7 +385,7 @@ public class BootstrapCheckTests extends ESTestCase {
             }
         };
 
-        RuntimeException e = expectThrows(
+        final RuntimeException e = expectThrows(
                 RuntimeException.class,
                 () -> BootstrapCheck.check(true, false, Collections.singletonList(check), "testClientJvmCheck"));
         assertThat(
@@ -395,8 +397,127 @@ public class BootstrapCheckTests extends ESTestCase {
         BootstrapCheck.check(true, false, Collections.singletonList(check), "testClientJvmCheck");
     }
 
+    public void testMightForkCheck() {
+        final AtomicBoolean isSeccompInstalled = new AtomicBoolean();
+        final AtomicBoolean mightFork = new AtomicBoolean();
+        final BootstrapCheck.MightForkCheck check = new BootstrapCheck.MightForkCheck() {
+            @Override
+            boolean isSeccompInstalled() {
+                return isSeccompInstalled.get();
+            }
+
+            @Override
+            boolean mightFork() {
+                return mightFork.get();
+            }
+
+            @Override
+            public String errorMessage() {
+                return "error";
+            }
+        };
+
+        runMightForkTest(
+            check,
+            isSeccompInstalled,
+            () -> mightFork.set(false),
+            () -> mightFork.set(true),
+            e -> assertThat(e.getMessage(), containsString("error")));
+    }
+
+    public void testOnErrorCheck() {
+        final AtomicBoolean isSeccompInstalled = new AtomicBoolean();
+        final AtomicReference<String> onError = new AtomicReference<>();
+        final BootstrapCheck.MightForkCheck check = new BootstrapCheck.OnErrorCheck() {
+            @Override
+            boolean isSeccompInstalled() {
+                return isSeccompInstalled.get();
+            }
+
+            @Override
+            String onError() {
+                return onError.get();
+            }
+        };
+
+        final String command = randomAsciiOfLength(16);
+        runMightForkTest(
+            check,
+            isSeccompInstalled,
+            () -> onError.set(randomBoolean() ? "" : null),
+            () -> onError.set(command),
+            e -> assertThat(
+                e.getMessage(),
+                containsString(
+                    "OnError [" + command + "] requires forking but is prevented by system call filters ([bootstrap.seccomp=true]);"
+                        + " upgrade to at least Java 8u92 and use ExitOnOutOfMemoryError")));
+    }
+
+    public void testOnOutOfMemoryErrorCheck() {
+        final AtomicBoolean isSeccompInstalled = new AtomicBoolean();
+        final AtomicReference<String> onOutOfMemoryError = new AtomicReference<>();
+        final BootstrapCheck.MightForkCheck check = new BootstrapCheck.OnOutOfMemoryErrorCheck() {
+            @Override
+            boolean isSeccompInstalled() {
+                return isSeccompInstalled.get();
+            }
+
+            @Override
+            String onOutOfMemoryError() {
+                return onOutOfMemoryError.get();
+            }
+        };
+
+        final String command = randomAsciiOfLength(16);
+        runMightForkTest(
+            check,
+            isSeccompInstalled,
+            () -> onOutOfMemoryError.set(randomBoolean() ? "" : null),
+            () -> onOutOfMemoryError.set(command),
+            e -> assertThat(
+                e.getMessage(),
+                containsString(
+                    "OnOutOfMemoryError [" + command + "]"
+                        + " requires forking but is prevented by system call filters ([bootstrap.seccomp=true]);"
+                        + " upgrade to at least Java 8u92 and use ExitOnOutOfMemoryError")));
+    }
+
+    private void runMightForkTest(
+        final BootstrapCheck.MightForkCheck check,
+        final AtomicBoolean isSeccompInstalled,
+        final Runnable disableMightFork,
+        final Runnable enableMightFork,
+        final Consumer<RuntimeException> consumer) {
+
+        // if seccomp is disabled, nothing should happen
+        isSeccompInstalled.set(false);
+        if (randomBoolean()) {
+            disableMightFork.run();
+        } else {
+            enableMightFork.run();
+        }
+        BootstrapCheck.check(true, randomBoolean(), Collections.singletonList(check), "testMightFork");
+
+        // if seccomp is enabled, but we will not fork, nothing should
+        // happen
+        isSeccompInstalled.set(true);
+        disableMightFork.run();
+        BootstrapCheck.check(true, randomBoolean(), Collections.singletonList(check), "testMightFork");
+
+        // if seccomp is enabled, and we might fork, the check should
+        // be enforced, regardless of bootstrap checks being enabled or
+        // not
+        isSeccompInstalled.set(true);
+        enableMightFork.run();
+
+        final RuntimeException e = expectThrows(
+            RuntimeException.class,
+            () -> BootstrapCheck.check(randomBoolean(), randomBoolean(), Collections.singletonList(check), "testMightFork"));
+        consumer.accept(e);
+    }
+
     public void testIgnoringSystemChecks() {
-        BootstrapCheck.Check check = new BootstrapCheck.Check() {
+        final BootstrapCheck.Check check = new BootstrapCheck.Check() {
             @Override
             public boolean check() {
                 return true;
@@ -430,4 +551,33 @@ public class BootstrapCheckTests extends ESTestCase {
         verify(logger).warn("error");
     }
 
+    public void testAlwaysEnforcedChecks() {
+        final BootstrapCheck.Check check = new BootstrapCheck.Check() {
+            @Override
+            public boolean check() {
+                return true;
+            }
+
+            @Override
+            public String errorMessage() {
+                return "error";
+            }
+
+            @Override
+            public boolean isSystemCheck() {
+                return randomBoolean();
+            }
+
+            @Override
+            public boolean alwaysEnforce() {
+                return true;
+            }
+        };
+
+        final RuntimeException alwaysEnforced = expectThrows(
+            RuntimeException.class,
+            () -> BootstrapCheck.check(randomBoolean(), randomBoolean(), Collections.singletonList(check), "testAlwaysEnforcedChecks"));
+        assertThat(alwaysEnforced, hasToString(containsString("error")));
+    }
+
 }

docs/reference/setup/bootstrap-checks.asciidoc

@@ -16,6 +16,10 @@ checks that fail appear as warnings in the Elasticsearch log. If
 Elasticsearch is in production mode, any bootstrap checks that fail will
 cause Elasticsearch to refuse to start.
 
+There are some bootstrap checks that are always enforced to prevent
+Elasticsearch from running with incompatible settings. These checks are
+documented individually.
+
 [float]
 === Development vs. production mode
 
@@ -152,3 +156,19 @@ JVM check, you must start Elasticsearch with the server VM. On modern
 systems and operating systems, the server VM is the
 default. Additionally, Elasticsearch is configured by default to force
 the server VM.
+
+=== OnError and OnOutOfMemoryError checks
+
+The JVM options `OnError` and `OnOutOfMemoryError` enable executing
+arbitrary commands if the JVM encounters a fatal error (`OnError`) or an
+`OutOfMemoryError` (`OnOutOfMemoryError`). However, by default,
+Elasticsearch system call filters (seccomp) are enabled and these
+filters prevent forking. Thus, using `OnError` or `OnOutOfMemoryError`
+and system call filters are incompatible. The `OnError` and
+`OnOutOfMemoryError` checks prevent Elasticsearch from starting if
+either of these JVM options are used and system call filters are
+enabled. This check is always enforced. To pass this check do not enable
+`OnError` nor `OnOutOfMemoryError`; instead, upgrade to Java 8u92 and
+use the JVM flag `ExitOnOutOfMemoryError`. While this does not have the
+full capabilities of `OnError` nor `OnOutOfMemoryError`, arbitrary
+forking will not be supported with seccomp enabled.