Autodetermine heap settings based on node roles and total system memory (#65905)

This commit expands our JVM egonomics to also automatically determine
appropriate heap size based on the total available system memory as well
as the roles assigned to the node. Role determination is done via a
naive parsing of elasticsearch.yml. No settings validation is done and
only the 'node.roles' setting is taken into consideration.

For heap purposes a node falls into one of four (4) categories:

1. A 'master-only' node. This is a node with only the 'master' role.
2. A 'ml-only' node. Similarly, a node with only the 'ml' role.
3. A 'data' node. This is basically the 'other' case. A node with any
set of roles other than only master or only ml is considered a 'data'
node, to include things like coordinating-only or "tie-breaker" nodes.
4. Unknown. This is the case if legacy settings are used. In this
scenario we fallback to the old default heap options of 1GB.

In all cases we short-circuit if a user provides explicit heap options
so we only ever auto-determine heap if no existing heap options exist.
Starting with this commit the default heap settings (1GB) are now
removed from the default jvm.options which means we'll start auto-
setting heap as the new default.

distribution/src/config/jvm.options

@@ -20,10 +20,13 @@
 ## IMPORTANT: JVM heap size
 ################################################################
 ##
-## You must always set the initial and maximum JVM heap size to
-## the same value. For example, to set the heap to 4 GB, create
-## a new file in the jvm.options.d directory containing these
-## lines:
+## The heap size is automatically configured by Elasticsearch
+## based on the available memory in your system and the roles
+## each node is configured to fulfill. If specifying heap is
+## required, it should be done through a file in jvm.options.d,
+## and the min and max should be set to the same value. For
+## example, to set the heap to 4 GB, create a new file in the
+## jvm.options.d directory containing these lines:
 ##
 ## -Xms4g
 ## -Xmx4g
@@ -33,13 +36,6 @@
 ##
 ################################################################
 
-# Xms represents the initial size of the JVM heap
-# Xmx represents the maximum size of the JVM heap
-
--Xms${heap.min}
--Xmx${heap.max}
-
-
 
 ################################################################
 ## Expert settings

distribution/tools/launchers/build.gradle

@@ -22,6 +22,7 @@ apply plugin: 'elasticsearch.build'
 
 dependencies {
   compileOnly project(':distribution:tools:java-version-checker')
+  compileOnly "org.yaml:snakeyaml:${versions.snakeyaml}"
   testImplementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   testImplementation "junit:junit:${versions.junit}"
   testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}"
@@ -44,4 +45,4 @@ tasks.named("testingConventions").configure {
 
 ["javadoc", "loggerUsageCheck", "jarHell"].each { tsk ->
   tasks.named(tsk).configure { enabled = false }
-}
+}

distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/DefaultSystemMemoryInfo.java

@@ -0,0 +1,51 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.tools.launchers;
+
+import com.sun.management.OperatingSystemMXBean;
+import org.elasticsearch.tools.java_version_checker.JavaVersion;
+import org.elasticsearch.tools.java_version_checker.SuppressForbidden;
+
+import java.lang.management.ManagementFactory;
+
+/**
+ * A {@link SystemMemoryInfo} which delegates to {@link OperatingSystemMXBean}.
+ *
+ * <p>Prior to JDK 14 {@link OperatingSystemMXBean} did not take into consideration container memory limits when reporting total system
+ * memory. Therefore attempts to use this implementation on earlier JDKs will result in an {@link SystemMemoryInfoException}.
+ */
+@SuppressForbidden(reason = "Using com.sun internals is the only way to query total system memory")
+public final class DefaultSystemMemoryInfo implements SystemMemoryInfo {
+    private final OperatingSystemMXBean operatingSystemMXBean;
+
+    public DefaultSystemMemoryInfo() {
+        this.operatingSystemMXBean = (OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();
+    }
+
+    @Override
+    @SuppressWarnings("deprecation")
+    public long availableSystemMemory() throws SystemMemoryInfoException {
+        if (JavaVersion.majorVersion(JavaVersion.CURRENT) < 14) {
+            throw new SystemMemoryInfoException("The minimum required Java version is 14 to use " + this.getClass().getName());
+        }
+
+        return operatingSystemMXBean.getTotalPhysicalMemorySize();
+    }
+}

distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmErgonomics.java

@@ -19,22 +19,13 @@
 
 package org.elasticsearch.tools.launchers;
 
-import java.io.BufferedReader;
 import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
-import java.util.Optional;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 /**
  * Tunes Elasticsearch JVM settings based on inspection of provided JVM options.
@@ -53,9 +44,9 @@ final class JvmErgonomics {
      */
     static List<String> choose(final List<String> userDefinedJvmOptions) throws InterruptedException, IOException {
         final List<String> ergonomicChoices = new ArrayList<>();
-        final Map<String, JvmOption> finalJvmOptions = finalJvmOptions(userDefinedJvmOptions);
-        final long heapSize = extractHeapSize(finalJvmOptions);
-        final long maxDirectMemorySize = extractMaxDirectMemorySize(finalJvmOptions);
+        final Map<String, JvmOption> finalJvmOptions = JvmOption.findFinalOptions(userDefinedJvmOptions);
+        final long heapSize = JvmOption.extractMaxHeapSize(finalJvmOptions);
+        final long maxDirectMemorySize = JvmOption.extractMaxDirectMemorySize(finalJvmOptions);
         if (maxDirectMemorySize == 0) {
             ergonomicChoices.add("-XX:MaxDirectMemorySize=" + heapSize / 2);
         }
@@ -78,89 +69,6 @@ final class JvmErgonomics {
         return ergonomicChoices;
     }
 
-    private static final Pattern OPTION = Pattern.compile(
-        "^\\s*\\S+\\s+(?<flag>\\S+)\\s+:?=\\s+(?<value>\\S+)?\\s+\\{[^}]+?\\}\\s+\\{(?<origin>[^}]+)}"
-    );
-
-    private static class JvmOption {
-        private final String value;
-        private final String origin;
-
-        JvmOption(String value, String origin) {
-            this.value = value;
-            this.origin = origin;
-        }
-
-        public Optional<String> getValue() {
-            return Optional.ofNullable(value);
-        }
-
-        public String getMandatoryValue() {
-            return value;
-        }
-
-        public boolean isCommandLineOrigin() {
-            return "command line".equals(this.origin);
-        }
-    }
-
-    static Map<String, JvmOption> finalJvmOptions(final List<String> userDefinedJvmOptions) throws InterruptedException, IOException {
-        return flagsFinal(userDefinedJvmOptions).stream()
-            .map(OPTION::matcher)
-            .filter(Matcher::matches)
-            .collect(Collectors.toUnmodifiableMap(m -> m.group("flag"), m -> new JvmOption(m.group("value"), m.group("origin"))));
-    }
-
-    private static List<String> flagsFinal(final List<String> userDefinedJvmOptions) throws InterruptedException, IOException {
-        /*
-         * To deduce the final set of JVM options that Elasticsearch is going to start with, we start a separate Java process with the JVM
-         * options that we would pass on the command line. For this Java process we will add two additional flags, -XX:+PrintFlagsFinal and
-         * -version. This causes the Java process that we start to parse the JVM options into their final values, display them on standard
-         * output, print the version to standard error, and then exit. The JVM itself never bootstraps, and therefore this process is
-         * lightweight. By doing this, we get the JVM options parsed exactly as the JVM that we are going to execute would parse them
-         * without having to implement our own JVM option parsing logic.
-         */
-        final String java = Path.of(System.getProperty("java.home"), "bin", "java").toString();
-        final List<String> command = Stream.of(
-            Stream.of(java),
-            userDefinedJvmOptions.stream(),
-            Stream.of("-Xshare:off"),
-            Stream.of("-XX:+PrintFlagsFinal"),
-            Stream.of("-version")
-        ).reduce(Stream::concat).get().collect(Collectors.toUnmodifiableList());
-        final Process process = new ProcessBuilder().command(command).start();
-        final List<String> output = readLinesFromInputStream(process.getInputStream());
-        final List<String> error = readLinesFromInputStream(process.getErrorStream());
-        final int status = process.waitFor();
-        if (status != 0) {
-            final String message = String.format(
-                Locale.ROOT,
-                "starting java failed with [%d]\noutput:\n%s\nerror:\n%s",
-                status,
-                String.join("\n", output),
-                String.join("\n", error)
-            );
-            throw new RuntimeException(message);
-        } else {
-            return output;
-        }
-    }
-
-    private static List<String> readLinesFromInputStream(final InputStream is) throws IOException {
-        try (InputStreamReader isr = new InputStreamReader(is, StandardCharsets.UTF_8); BufferedReader br = new BufferedReader(isr)) {
-            return br.lines().collect(Collectors.toUnmodifiableList());
-        }
-    }
-
-    // package private for testing
-    static Long extractHeapSize(final Map<String, JvmOption> finalJvmOptions) {
-        return Long.parseLong(finalJvmOptions.get("MaxHeapSize").getMandatoryValue());
-    }
-
-    static long extractMaxDirectMemorySize(final Map<String, JvmOption> finalJvmOptions) {
-        return Long.parseLong(finalJvmOptions.get("MaxDirectMemorySize").getMandatoryValue());
-    }
-
     // Tune G1GC options for heaps < 8GB
     static boolean tuneG1GCForSmallHeap(final long heapSize) {
         return heapSize < 8L << 30;

distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOption.java

@@ -0,0 +1,136 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.tools.launchers;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+class JvmOption {
+    private final String value;
+    private final String origin;
+
+    JvmOption(String value, String origin) {
+        this.value = value;
+        this.origin = origin;
+    }
+
+    public Optional<String> getValue() {
+        return Optional.ofNullable(value);
+    }
+
+    public String getMandatoryValue() {
+        return value;
+    }
+
+    public boolean isCommandLineOrigin() {
+        return "command line".equals(this.origin);
+    }
+
+    private static final Pattern OPTION = Pattern.compile(
+        "^\\s*\\S+\\s+(?<flag>\\S+)\\s+:?=\\s+(?<value>\\S+)?\\s+\\{[^}]+?\\}\\s+\\{(?<origin>[^}]+)}"
+    );
+
+    public static Long extractMaxHeapSize(final Map<String, JvmOption> finalJvmOptions) {
+        return Long.parseLong(finalJvmOptions.get("MaxHeapSize").getMandatoryValue());
+    }
+
+    public static boolean isMaxHeapSpecified(final Map<String, JvmOption> finalJvmOptions) {
+        JvmOption maxHeapSize = finalJvmOptions.get("MaxHeapSize");
+        return maxHeapSize != null && maxHeapSize.isCommandLineOrigin();
+    }
+
+    public static boolean isMinHeapSpecified(final Map<String, JvmOption> finalJvmOptions) {
+        JvmOption minHeapSize = finalJvmOptions.get("MinHeapSize");
+        return minHeapSize != null && minHeapSize.isCommandLineOrigin();
+    }
+
+    public static boolean isInitialHeapSpecified(final Map<String, JvmOption> finalJvmOptions) {
+        JvmOption initialHeapSize = finalJvmOptions.get("InitialHeapSize");
+        return initialHeapSize != null && initialHeapSize.isCommandLineOrigin();
+    }
+
+    public static long extractMaxDirectMemorySize(final Map<String, JvmOption> finalJvmOptions) {
+        return Long.parseLong(finalJvmOptions.get("MaxDirectMemorySize").getMandatoryValue());
+    }
+
+    /**
+     * Determine the options present when invoking a JVM with the given user defined options.
+     */
+    public static Map<String, JvmOption> findFinalOptions(final List<String> userDefinedJvmOptions) throws InterruptedException,
+        IOException {
+        return flagsFinal(userDefinedJvmOptions).stream()
+            .map(OPTION::matcher)
+            .filter(Matcher::matches)
+            .collect(Collectors.toUnmodifiableMap(m -> m.group("flag"), m -> new JvmOption(m.group("value"), m.group("origin"))));
+    }
+
+    private static List<String> flagsFinal(final List<String> userDefinedJvmOptions) throws InterruptedException, IOException {
+        /*
+         * To deduce the final set of JVM options that Elasticsearch is going to start with, we start a separate Java process with the JVM
+         * options that we would pass on the command line. For this Java process we will add two additional flags, -XX:+PrintFlagsFinal and
+         * -version. This causes the Java process that we start to parse the JVM options into their final values, display them on standard
+         * output, print the version to standard error, and then exit. The JVM itself never bootstraps, and therefore this process is
+         * lightweight. By doing this, we get the JVM options parsed exactly as the JVM that we are going to execute would parse them
+         * without having to implement our own JVM option parsing logic.
+         */
+        final String java = Path.of(System.getProperty("java.home"), "bin", "java").toString();
+        final List<String> command = Stream.of(
+            Stream.of(java),
+            userDefinedJvmOptions.stream(),
+            Stream.of("-Xshare:off"),
+            Stream.of("-XX:+PrintFlagsFinal"),
+            Stream.of("-version")
+        ).reduce(Stream::concat).get().collect(Collectors.toUnmodifiableList());
+        final Process process = new ProcessBuilder().command(command).start();
+        final List<String> output = readLinesFromInputStream(process.getInputStream());
+        final List<String> error = readLinesFromInputStream(process.getErrorStream());
+        final int status = process.waitFor();
+        if (status != 0) {
+            final String message = String.format(
+                Locale.ROOT,
+                "starting java failed with [%d]\noutput:\n%s\nerror:\n%s",
+                status,
+                String.join("\n", output),
+                String.join("\n", error)
+            );
+            throw new RuntimeException(message);
+        } else {
+            return output;
+        }
+    }
+
+    private static List<String> readLinesFromInputStream(final InputStream is) throws IOException {
+        try (InputStreamReader isr = new InputStreamReader(is, StandardCharsets.UTF_8); BufferedReader br = new BufferedReader(isr)) {
+            return br.lines().collect(Collectors.toUnmodifiableList());
+        }
+    }
+}

distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOptionsParser.java

@@ -134,6 +134,7 @@ final class JvmOptionsParser {
         throws InterruptedException, IOException, JvmOptionsFileParserException {
 
         final List<String> jvmOptions = readJvmOptionsFiles(config);
+        final MachineDependentHeap machineDependentHeap = new MachineDependentHeap(new DefaultSystemMemoryInfo());
 
         if (esJavaOpts != null) {
             jvmOptions.addAll(
@@ -142,6 +143,7 @@ final class JvmOptionsParser {
         }
 
         final List<String> substitutedJvmOptions = substitutePlaceholders(jvmOptions, Collections.unmodifiableMap(substitutions));
+        substitutedJvmOptions.addAll(machineDependentHeap.determineHeapSettings(config, substitutedJvmOptions));
         final List<String> ergonomicJvmOptions = JvmErgonomics.choose(substitutedJvmOptions);
         final List<String> systemJvmOptions = SystemJvmOptions.systemJvmOptions();
         final List<String> bootstrapOptions = BootstrapJvmOptions.bootstrapJvmOptions(plugins);

distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/MachineDependentHeap.java

@@ -0,0 +1,250 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.tools.launchers;
+
+import org.yaml.snakeyaml.Yaml;
+import org.yaml.snakeyaml.error.YAMLException;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
+
+import static java.lang.Math.max;
+import static java.lang.Math.min;
+import static org.elasticsearch.tools.launchers.JvmOption.isInitialHeapSpecified;
+import static org.elasticsearch.tools.launchers.JvmOption.isMaxHeapSpecified;
+import static org.elasticsearch.tools.launchers.JvmOption.isMinHeapSpecified;
+
+/**
+ * Determines optimal default heap settings based on available system memory and assigned node roles.
+ */
+public final class MachineDependentHeap {
+    private static final long GB = 1024L * 1024L * 1024L; // 1GB
+    private static final long MAX_HEAP_SIZE = GB * 31; // 31GB
+    private static final long MAX_ML_HEAP_SIZE = GB * 2; // 2GB
+    private static final long MIN_HEAP_SIZE = 1024 * 1024 * 128; // 128MB
+    private static final int DEFAULT_HEAP_SIZE_MB = 1024;
+    private static final String ELASTICSEARCH_YML = "elasticsearch.yml";
+
+    private final SystemMemoryInfo systemMemoryInfo;
+
+    public MachineDependentHeap(SystemMemoryInfo systemMemoryInfo) {
+        this.systemMemoryInfo = systemMemoryInfo;
+    }
+
+    /**
+     * Calculate heap options.
+     *
+     * @param configDir path to config directory
+     * @param userDefinedJvmOptions JVM arguments provided by the user
+     * @return final heap options, or an empty collection if user provided heap options are to be used
+     * @throws IOException if unable to load elasticsearch.yml
+     */
+    public List<String> determineHeapSettings(Path configDir, List<String> userDefinedJvmOptions) throws IOException, InterruptedException {
+        // TODO: this could be more efficient, to only parse final options once
+        final Map<String, JvmOption> finalJvmOptions = JvmOption.findFinalOptions(userDefinedJvmOptions);
+        if (isMaxHeapSpecified(finalJvmOptions) || isMinHeapSpecified(finalJvmOptions) || isInitialHeapSpecified(finalJvmOptions)) {
+            // User has explicitly set memory settings so we use those
+            return Collections.emptyList();
+        }
+
+        Path config = configDir.resolve(ELASTICSEARCH_YML);
+        try (InputStream in = Files.newInputStream(config)) {
+            return determineHeapSettings(in);
+        }
+    }
+
+    List<String> determineHeapSettings(InputStream config) {
+        MachineNodeRole nodeRole = NodeRoleParser.parse(config);
+
+        try {
+            long availableSystemMemory = systemMemoryInfo.availableSystemMemory();
+            return options(nodeRole.heap(availableSystemMemory));
+        } catch (SystemMemoryInfo.SystemMemoryInfoException e) {
+            // If unable to determine system memory (ex: incompatible jdk version) fallback to defaults
+            return options(DEFAULT_HEAP_SIZE_MB);
+        }
+    }
+
+    private static List<String> options(int heapSize) {
+        return List.of("-Xms" + heapSize + "m", "-Xmx" + heapSize + "m");
+    }
+
+    /**
+     * Parses role information from elasticsearch.yml and determines machine node role.
+     */
+    static class NodeRoleParser {
+        private static final Set<String> LEGACY_ROLE_SETTINGS = Set.of(
+            "node.master",
+            "node.ingest",
+            "node.data",
+            "node.voting_only",
+            "node.ml",
+            "node.transform",
+            "node.remote_cluster_client"
+        );
+
+        @SuppressWarnings("unchecked")
+        public static MachineNodeRole parse(InputStream config) {
+            Yaml yaml = new Yaml();
+            Map<String, Object> root;
+            try {
+                root = yaml.load(config);
+            } catch (YAMLException | ClassCastException ex) {
+                // Strangely formatted config, so just return defaults and let startup settings validation catch the problem
+                return MachineNodeRole.UNKNOWN;
+            }
+
+            if (root != null) {
+                Map<String, Object> map = flatten(root, null);
+
+                if (hasLegacySettings(map.keySet())) {
+                    // We don't attempt to auto-determine heap if legacy role settings are used
+                    return MachineNodeRole.UNKNOWN;
+                } else {
+                    List<String> roles = null;
+                    try {
+                        if (map.containsKey("node.roles")) {
+                            roles = (List<String>) map.get("node.roles");
+                        }
+                    } catch (ClassCastException ex) {
+                        return MachineNodeRole.UNKNOWN;
+                    }
+
+                    if (roles == null || roles.isEmpty()) {
+                        // If roles are missing or empty (coordinating node) assume defaults and consider this a data node
+                        return MachineNodeRole.DATA;
+                    } else if (containsOnly(roles, "master")) {
+                        return MachineNodeRole.MASTER_ONLY;
+                    } else if (containsOnly(roles, "ml")) {
+                        return MachineNodeRole.ML_ONLY;
+                    } else {
+                        return MachineNodeRole.DATA;
+                    }
+                }
+            } else { // if the config is completely empty, then assume defaults and consider this a data node
+                return MachineNodeRole.DATA;
+            }
+        }
+
+        /**
+         * Flattens a nested configuration structure. This creates a consistent way of referencing settings from a config file that uses
+         * a mix of object and flat setting notation. The returned map is a single-level deep structure of dot-notation property names
+         * to values.
+         *
+         * <p>No attempt is made to deterministically deal with duplicate settings, nor are they explicitly disallowed.
+         *
+         * @param config nested configuration map
+         * @param parentPath parent node path or {@code null} if parsing the root node
+         * @return flattened configuration map
+         */
+        @SuppressWarnings("unchecked")
+        private static Map<String, Object> flatten(Map<String, Object> config, String parentPath) {
+            Map<String, Object> flatMap = new HashMap<>();
+            String prefix = parentPath != null ? parentPath + "." : "";
+
+            for (Map.Entry<String, Object> entry : config.entrySet()) {
+                if (entry.getValue() instanceof Map) {
+                    flatMap.putAll(flatten((Map<String, Object>) entry.getValue(), prefix + entry.getKey()));
+                } else {
+                    flatMap.put(prefix + entry.getKey(), entry.getValue());
+                }
+            }
+
+            return flatMap;
+        }
+
+        @SuppressWarnings("unchecked")
+        private static <T> boolean containsOnly(Collection<T> collection, T... items) {
+            return Arrays.asList(items).containsAll(collection);
+        }
+
+        private static boolean hasLegacySettings(Set<String> keys) {
+            return LEGACY_ROLE_SETTINGS.stream().anyMatch(keys::contains);
+        }
+    }
+
+    enum MachineNodeRole {
+        /**
+         * Master-only node.
+         *
+         * <p>Heap is computed as 60% of total system memory up to a maximum of 31 gigabytes.
+         */
+        MASTER_ONLY(m -> mb(min((long) (m * .6), MAX_HEAP_SIZE))),
+
+        /**
+         * Machine learning only node.
+         *
+         * <p>Heap is computed as:
+         * <ul>
+         *     <li>40% of total system memory when less than 2 gigabytes.</li>
+         *     <li>25% of total system memory when greater than 2 gigabytes up to a maximum of 2 gigabytes.</li>
+         * </ul>
+         */
+        ML_ONLY(m -> mb(m < (GB * 2) ? (long) (m * .4) : (long) min(m * .25, MAX_ML_HEAP_SIZE))),
+
+        /**
+         * Data node. Essentially any node that isn't a master or ML only node.
+         *
+         * <p>Heap is computed as:
+         * <ul>
+         *     <li>40% of total system memory when less than 1 gigabyte with a minimum of 128 megabytes.</li>
+         *     <li>50% of total system memory when greater than 1 gigabyte up to a maximum of 31 gigabytes.</li>
+         * </ul>
+         */
+        DATA(m -> mb(m < GB ? max((long) (m * .4), MIN_HEAP_SIZE) : min((long) (m * .5), MAX_HEAP_SIZE))),
+
+        /**
+         * Unknown role node.
+         *
+         * <p>Hard-code heap to a default of 1 gigabyte.
+         */
+        UNKNOWN(m -> DEFAULT_HEAP_SIZE_MB);
+
+        private final Function<Long, Integer> formula;
+
+        MachineNodeRole(Function<Long, Integer> formula) {
+            this.formula = formula;
+        }
+
+        /**
+         * Determine the appropriate heap size for the given role and available system memory.
+         *
+         * @param systemMemory total available system memory in bytes
+         * @return recommended heap size in megabytes
+         */
+        public int heap(long systemMemory) {
+            return formula.apply(systemMemory);
+        }
+
+        private static int mb(long bytes) {
+            return (int) (bytes / (1024 * 1024));
+        }
+    }
+}

distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/SystemMemoryInfo.java

@@ -0,0 +1,42 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.tools.launchers;
+
+/**
+ * Determines available system memory that could be allocated for Elasticsearch, to include JVM heap and other native processes.
+ * The "available system memory" is defined as the total system memory which is visible to the Elasticsearch process. For instances
+ * in which Elasticsearch is running in a containerized environment (i.e. Docker) this is expected to be the limits set for the container,
+ * not the host system.
+ */
+public interface SystemMemoryInfo {
+
+    /**
+     *
+     * @return total system memory available to heap or native process allocation in bytes
+     * @throws SystemMemoryInfoException if unable to determine available system memory
+     */
+    long availableSystemMemory() throws SystemMemoryInfoException;
+
+    class SystemMemoryInfoException extends Exception {
+        public SystemMemoryInfoException(String message) {
+            super(message);
+        }
+    }
+}

distribution/tools/launchers/src/test/java/org/elasticsearch/tools/launchers/JvmErgonomicsTests.java

@@ -42,23 +42,23 @@ import static org.junit.Assert.fail;
 public class JvmErgonomicsTests extends LaunchersTestCase {
 
     public void testExtractValidHeapSizeUsingXmx() throws InterruptedException, IOException {
-        assertThat(JvmErgonomics.extractHeapSize(JvmErgonomics.finalJvmOptions(Collections.singletonList("-Xmx2g"))), equalTo(2L << 30));
+        assertThat(JvmOption.extractMaxHeapSize(JvmOption.findFinalOptions(Collections.singletonList("-Xmx2g"))), equalTo(2L << 30));
     }
 
     public void testExtractValidHeapSizeUsingMaxHeapSize() throws InterruptedException, IOException {
         assertThat(
-            JvmErgonomics.extractHeapSize(JvmErgonomics.finalJvmOptions(Collections.singletonList("-XX:MaxHeapSize=2g"))),
+            JvmOption.extractMaxHeapSize(JvmOption.findFinalOptions(Collections.singletonList("-XX:MaxHeapSize=2g"))),
             equalTo(2L << 30)
         );
     }
 
     public void testExtractValidHeapSizeNoOptionPresent() throws InterruptedException, IOException {
-        assertThat(JvmErgonomics.extractHeapSize(JvmErgonomics.finalJvmOptions(Collections.emptyList())), greaterThan(0L));
+        assertThat(JvmOption.extractMaxHeapSize(JvmOption.findFinalOptions(Collections.emptyList())), greaterThan(0L));
     }
 
     public void testHeapSizeInvalid() throws InterruptedException, IOException {
         try {
-            JvmErgonomics.extractHeapSize(JvmErgonomics.finalJvmOptions(Collections.singletonList("-Xmx2Z")));
+            JvmOption.extractMaxHeapSize(JvmOption.findFinalOptions(Collections.singletonList("-Xmx2Z")));
             fail("expected starting java to fail");
         } catch (final RuntimeException e) {
             assertThat(e, hasToString(containsString(("starting java failed"))));
@@ -68,7 +68,7 @@ public class JvmErgonomicsTests extends LaunchersTestCase {
 
     public void testHeapSizeTooSmall() throws InterruptedException, IOException {
         try {
-            JvmErgonomics.extractHeapSize(JvmErgonomics.finalJvmOptions(Collections.singletonList("-Xmx1024")));
+            JvmOption.extractMaxHeapSize(JvmOption.findFinalOptions(Collections.singletonList("-Xmx1024")));
             fail("expected starting java to fail");
         } catch (final RuntimeException e) {
             assertThat(e, hasToString(containsString(("starting java failed"))));
@@ -78,7 +78,7 @@ public class JvmErgonomicsTests extends LaunchersTestCase {
 
     public void testHeapSizeWithSpace() throws InterruptedException, IOException {
         try {
-            JvmErgonomics.extractHeapSize(JvmErgonomics.finalJvmOptions(Collections.singletonList("-Xmx 1024")));
+            JvmOption.extractMaxHeapSize(JvmOption.findFinalOptions(Collections.singletonList("-Xmx 1024")));
             fail("expected starting java to fail");
         } catch (final RuntimeException e) {
             assertThat(e, hasToString(containsString(("starting java failed"))));
@@ -87,17 +87,12 @@ public class JvmErgonomicsTests extends LaunchersTestCase {
     }
 
     public void testMaxDirectMemorySizeUnset() throws InterruptedException, IOException {
-        assertThat(
-            JvmErgonomics.extractMaxDirectMemorySize(JvmErgonomics.finalJvmOptions(Collections.singletonList("-Xmx1g"))),
-            equalTo(0L)
-        );
+        assertThat(JvmOption.extractMaxDirectMemorySize(JvmOption.findFinalOptions(Collections.singletonList("-Xmx1g"))), equalTo(0L));
     }
 
     public void testMaxDirectMemorySizeSet() throws InterruptedException, IOException {
         assertThat(
-            JvmErgonomics.extractMaxDirectMemorySize(
-                JvmErgonomics.finalJvmOptions(Arrays.asList("-Xmx1g", "-XX:MaxDirectMemorySize=512m"))
-            ),
+            JvmOption.extractMaxDirectMemorySize(JvmOption.findFinalOptions(Arrays.asList("-Xmx1g", "-XX:MaxDirectMemorySize=512m"))),
             equalTo(512L << 20)
         );
     }

distribution/tools/launchers/src/test/java/org/elasticsearch/tools/launchers/MachineDependentHeapTests.java

@@ -0,0 +1,127 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.tools.launchers;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Collections;
+import java.util.List;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.empty;
+import static org.junit.Assert.assertThat;
+
+public class MachineDependentHeapTests extends LaunchersTestCase {
+
+    public void testDefaultHeapSize() throws Exception {
+        MachineDependentHeap heap = new MachineDependentHeap(systemMemoryInGigabytes(8));
+        List<String> options = heap.determineHeapSettings(configPath(), Collections.emptyList());
+        assertThat(options, containsInAnyOrder("-Xmx4096m", "-Xms4096m"));
+    }
+
+    public void testUserPassedHeapArgs() throws Exception {
+        MachineDependentHeap heap = new MachineDependentHeap(systemMemoryInGigabytes(8));
+        List<String> options = heap.determineHeapSettings(configPath(), List.of("-Xmx4g"));
+        assertThat(options, empty());
+
+        options = heap.determineHeapSettings(configPath(), List.of("-Xms4g"));
+        assertThat(options, empty());
+    }
+
+    public void testMasterOnlyOptions() {
+        List<String> options = calculateHeap(16, "master");
+        assertThat(options, containsInAnyOrder("-Xmx9830m", "-Xms9830m"));
+
+        options = calculateHeap(64, "master");
+        assertThat(options, containsInAnyOrder("-Xmx31744m", "-Xms31744m"));
+    }
+
+    public void testMlOnlyOptions() {
+        List<String> options = calculateHeap(1, "ml");
+        assertThat(options, containsInAnyOrder("-Xmx409m", "-Xms409m"));
+
+        options = calculateHeap(4, "ml");
+        assertThat(options, containsInAnyOrder("-Xmx1024m", "-Xms1024m"));
+
+        options = calculateHeap(32, "ml");
+        assertThat(options, containsInAnyOrder("-Xmx2048m", "-Xms2048m"));
+    }
+
+    public void testDataNodeOptions() {
+        List<String> options = calculateHeap(1, "data");
+        assertThat(options, containsInAnyOrder("-Xmx512m", "-Xms512m"));
+
+        options = calculateHeap(8, "data");
+        assertThat(options, containsInAnyOrder("-Xmx4096m", "-Xms4096m"));
+
+        options = calculateHeap(64, "data");
+        assertThat(options, containsInAnyOrder("-Xmx31744m", "-Xms31744m"));
+
+        options = calculateHeap(0.5, "data");
+        assertThat(options, containsInAnyOrder("-Xmx204m", "-Xms204m"));
+
+        options = calculateHeap(0.2, "data");
+        assertThat(options, containsInAnyOrder("-Xmx128m", "-Xms128m"));
+    }
+
+    public void testFallbackOptions() throws Exception {
+        MachineDependentHeap machineDependentHeap = new MachineDependentHeap(errorThrowingMemoryInfo());
+        List<String> options = machineDependentHeap.determineHeapSettings(configPath(), Collections.emptyList());
+        assertThat(options, containsInAnyOrder("-Xmx1024m", "-Xms1024m"));
+    }
+
+    private static List<String> calculateHeap(double memoryInGigabytes, String... roles) {
+        MachineDependentHeap machineDependentHeap = new MachineDependentHeap(systemMemoryInGigabytes(memoryInGigabytes));
+        String configYaml = "node.roles: [" + String.join(",", roles) + "]";
+        return calculateHeap(machineDependentHeap, configYaml);
+    }
+
+    private static List<String> calculateHeap(MachineDependentHeap machineDependentHeap, String configYaml) {
+        try (InputStream in = new ByteArrayInputStream(configYaml.getBytes(StandardCharsets.UTF_8))) {
+            return machineDependentHeap.determineHeapSettings(in);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    private static SystemMemoryInfo systemMemoryInGigabytes(double gigabytes) {
+        return () -> (long) (gigabytes * 1024 * 1024 * 1024);
+    }
+
+    private static SystemMemoryInfo errorThrowingMemoryInfo() {
+        return () -> { throw new SystemMemoryInfo.SystemMemoryInfoException("something went wrong"); };
+    }
+
+    private static Path configPath() {
+        URL resource = MachineDependentHeapTests.class.getResource("/config/elasticsearch.yml");
+        try {
+            return Paths.get(resource.toURI()).getParent();
+        } catch (URISyntaxException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

distribution/tools/launchers/src/test/java/org/elasticsearch/tools/launchers/NodeRoleParserTests.java

@@ -0,0 +1,118 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.tools.launchers;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.function.Consumer;
+
+import static org.elasticsearch.tools.launchers.MachineDependentHeap.MachineNodeRole.DATA;
+import static org.elasticsearch.tools.launchers.MachineDependentHeap.MachineNodeRole.MASTER_ONLY;
+import static org.elasticsearch.tools.launchers.MachineDependentHeap.MachineNodeRole.ML_ONLY;
+import static org.elasticsearch.tools.launchers.MachineDependentHeap.MachineNodeRole.UNKNOWN;
+import static org.hamcrest.CoreMatchers.not;
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.Assert.assertThat;
+
+public class NodeRoleParserTests extends LaunchersTestCase {
+
+    public void testMasterOnlyNode() throws IOException {
+        MachineDependentHeap.MachineNodeRole nodeRole = parseConfig(sb -> sb.append("node.roles: [master]"));
+        assertThat(nodeRole, equalTo(MASTER_ONLY));
+
+        nodeRole = parseConfig(sb -> sb.append("node.roles: [master, some_other_role]"));
+        assertThat(nodeRole, not(equalTo(MASTER_ONLY)));
+    }
+
+    public void testMlOnlyNode() throws IOException {
+        MachineDependentHeap.MachineNodeRole nodeRole = parseConfig(sb -> sb.append("node.roles: [ml]"));
+        assertThat(nodeRole, equalTo(ML_ONLY));
+
+        nodeRole = parseConfig(sb -> sb.append("node.roles: [ml, some_other_role]"));
+        assertThat(nodeRole, not(equalTo(ML_ONLY)));
+    }
+
+    public void testDataNode() throws IOException {
+        MachineDependentHeap.MachineNodeRole nodeRole = parseConfig(sb -> {});
+        assertThat(nodeRole, equalTo(DATA));
+
+        nodeRole = parseConfig(sb -> sb.append("node.roles: []"));
+        assertThat(nodeRole, equalTo(DATA));
+
+        nodeRole = parseConfig(sb -> sb.append("node.roles: [some_unknown_role]"));
+        assertThat(nodeRole, equalTo(DATA));
+
+        nodeRole = parseConfig(sb -> sb.append("node.roles: [master, ingest]"));
+        assertThat(nodeRole, equalTo(DATA));
+
+        nodeRole = parseConfig(sb -> sb.append("node.roles: [ml, master]"));
+        assertThat(nodeRole, equalTo(DATA));
+    }
+
+    public void testLegacySettings() throws IOException {
+        MachineDependentHeap.MachineNodeRole nodeRole = parseConfig(sb -> sb.append("node.ml: true"));
+        assertThat(nodeRole, equalTo(UNKNOWN));
+
+        nodeRole = parseConfig(sb -> sb.append("node.master: true"));
+        assertThat(nodeRole, equalTo(UNKNOWN));
+
+        nodeRole = parseConfig(sb -> sb.append("node.data: false"));
+        assertThat(nodeRole, equalTo(UNKNOWN));
+
+        nodeRole = parseConfig(sb -> sb.append("node.ingest: false"));
+        assertThat(nodeRole, equalTo(UNKNOWN));
+    }
+
+    public void testYamlSyntax() throws IOException {
+        MachineDependentHeap.MachineNodeRole nodeRole = parseConfig(sb -> {
+            sb.append("node:\n");
+            sb.append("  roles:\n");
+            sb.append("    - master");
+        });
+        assertThat(nodeRole, equalTo(MASTER_ONLY));
+
+        nodeRole = parseConfig(sb -> {
+            sb.append("node:\n");
+            sb.append("  roles: [ml]");
+        });
+        assertThat(nodeRole, equalTo(ML_ONLY));
+    }
+
+    public void testInvalidYaml() throws IOException {
+        MachineDependentHeap.MachineNodeRole nodeRole = parseConfig(sb -> sb.append("notyaml"));
+        assertThat(nodeRole, equalTo(UNKNOWN));
+    }
+
+    public void testInvalidRoleSyntax() throws IOException {
+        MachineDependentHeap.MachineNodeRole nodeRole = parseConfig(sb -> sb.append("node.roles: foo"));
+        assertThat(nodeRole, equalTo(UNKNOWN));
+    }
+
+    private static MachineDependentHeap.MachineNodeRole parseConfig(Consumer<StringBuilder> action) throws IOException {
+        StringBuilder sb = new StringBuilder();
+        action.accept(sb);
+
+        try (InputStream config = new ByteArrayInputStream(sb.toString().getBytes(StandardCharsets.UTF_8))) {
+            return MachineDependentHeap.NodeRoleParser.parse(config);
+        }
+    }
+}

distribution/tools/launchers/src/test/resources/config/elasticsearch.yml

@@ -0,0 +1 @@
+node.roles: []

qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java

@@ -248,7 +248,8 @@ public class ArchiveTests extends PackagingTestCase {
     public void test70CustomPathConfAndJvmOptions() throws Exception {
 
         withCustomConfig(tempConf -> {
-            final List<String> jvmOptions = List.of("-Xms512m", "-Xmx512m", "-Dlog4j2.disable.jmx=true");
+            setHeap("512m", tempConf);
+            final List<String> jvmOptions = List.of("-Dlog4j2.disable.jmx=true");
             Files.write(tempConf.resolve("jvm.options"), jvmOptions, CREATE, APPEND);
 
             sh.getEnv().put("ES_JAVA_OPTS", "-XX:-UseCompressedOops");
@@ -266,6 +267,7 @@ public class ArchiveTests extends PackagingTestCase {
     public void test71CustomJvmOptionsDirectoryFile() throws Exception {
         final Path heapOptions = installation.config(Paths.get("jvm.options.d", "heap.options"));
         try {
+            setHeap(null); // delete default options
             append(heapOptions, "-Xms512m\n-Xmx512m\n");
 
             startElasticsearch();
@@ -283,6 +285,7 @@ public class ArchiveTests extends PackagingTestCase {
         final Path firstOptions = installation.config(Paths.get("jvm.options.d", "first.options"));
         final Path secondOptions = installation.config(Paths.get("jvm.options.d", "second.options"));
         try {
+            setHeap(null); // delete default options
             /*
              * We override the heap in the first file, and disable compressed oops, and override the heap in the second file. By doing this,
              * we can test that both files are processed by the JVM options parser, and also that they are processed in lexicographic order.
@@ -306,13 +309,10 @@ public class ArchiveTests extends PackagingTestCase {
     public void test73CustomJvmOptionsDirectoryFilesWithoutOptionsExtensionIgnored() throws Exception {
         final Path jvmOptionsIgnored = installation.config(Paths.get("jvm.options.d", "jvm.options.ignored"));
         try {
-            append(jvmOptionsIgnored, "-Xms512\n-Xmx512m\n");
+            append(jvmOptionsIgnored, "-Xthis_is_not_a_valid_option\n");
 
             startElasticsearch();
-
-            final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes"));
-            assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":1073741824"));
-
+            ServerUtils.runElasticsearchTests();
             stopElasticsearch();
         } finally {
             rm(jvmOptionsIgnored);

qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.packaging.test;
 
 import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.http.client.fluent.Request;
 import org.elasticsearch.packaging.util.Distribution;
 import org.elasticsearch.packaging.util.Installation;
@@ -35,9 +36,11 @@ import org.junit.BeforeClass;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -68,6 +71,7 @@ import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.emptyString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.hasItems;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.matchesPattern;
@@ -732,6 +736,45 @@ public class DockerTests extends PackagingTestCase {
         assertThat("Failed to find [cpuacct] in node OS cgroup stats", cgroupStats.get("cpuacct"), not(nullValue()));
     }
 
+    /**
+     * Check that when available system memory is constrained by Docker, the machine-dependant heap sizing
+     * logic sets the correct heap size, based on the container limits.
+     */
+    public void test150MachineDependentHeap() throws Exception {
+        // Start by ensuring `jvm.options` doesn't define any heap options
+        final Path jvmOptionsPath = tempDir.resolve("jvm.options");
+        final Path containerJvmOptionsPath = installation.config("jvm.options");
+        copyFromContainer(containerJvmOptionsPath, jvmOptionsPath);
+
+        final List<String> jvmOptions = Files.readAllLines(jvmOptionsPath)
+            .stream()
+            .filter(line -> (line.startsWith("-Xms") || line.startsWith("-Xmx")) == false)
+            .collect(Collectors.toList());
+
+        Files.writeString(jvmOptionsPath, String.join("\n", jvmOptions));
+
+        // Now run the container, being explicit about the available memory
+        runContainer(distribution(), builder().memory("942m").volumes(Map.of(jvmOptionsPath, containerJvmOptionsPath)));
+        waitForElasticsearch(installation);
+
+        // Grab the container output and find the line where it print the JVM arguments. This will
+        // let us see what the automatic heap sizing calculated.
+        final Optional<String> jvmArgumentsLine = getContainerLogs().stdout.lines()
+            .filter(line -> line.contains("JVM arguments"))
+            .findFirst();
+        assertThat("Failed to find jvmArguments in container logs", jvmArgumentsLine.isPresent(), is(true));
+
+        final JsonNode jsonNode = new ObjectMapper().readTree(jvmArgumentsLine.get());
+
+        final String argsStr = jsonNode.get("message").textValue();
+        final List<String> xArgs = Arrays.stream(argsStr.substring(1, argsStr.length() - 1).split(",\\s*"))
+            .filter(arg -> arg.startsWith("-X"))
+            .collect(Collectors.toList());
+
+        // This is roughly 0.4 * 942
+        assertThat(xArgs, hasItems("-Xms376m", "-Xmx376m"));
+    }
+
     /**
      * Check that the UBI images has the correct license information in the correct place.
      */

qa/os/src/test/java/org/elasticsearch/packaging/test/PackageTests.java

@@ -133,31 +133,14 @@ public class PackageTests extends PackagingTestCase {
     }
 
     public void test34CustomJvmOptionsDirectoryFile() throws Exception {
-        final Path heapOptions = installation.config(Paths.get("jvm.options.d", "heap.options"));
-        try {
-            append(heapOptions, "-Xms512m\n-Xmx512m\n");
-
-            startElasticsearch();
+        setHeap("512m");
 
-            final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes"));
-            assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":536870912"));
-
-            stopElasticsearch();
-        } finally {
-            rm(heapOptions);
-        }
-    }
+        startElasticsearch();
 
-    public void test42BundledJdkRemoved() throws Exception {
-        assumeThat(distribution().hasJdk, is(true));
+        final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes"));
+        assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":536870912"));
 
-        Path relocatedJdk = installation.bundledJdk.getParent().resolve("jdk.relocated");
-        try {
-            mv(installation.bundledJdk, relocatedJdk);
-            assertRunsWithJavaHome();
-        } finally {
-            mv(relocatedJdk, installation.bundledJdk);
-        }
+        stopElasticsearch();
     }
 
     public void test40StartServer() throws Exception {
@@ -177,6 +160,18 @@ public class PackageTests extends PackagingTestCase {
         stopElasticsearch();
     }
 
+    public void test42BundledJdkRemoved() throws Exception {
+        assumeThat(distribution().hasJdk, is(true));
+
+        Path relocatedJdk = installation.bundledJdk.getParent().resolve("jdk.relocated");
+        try {
+            mv(installation.bundledJdk, relocatedJdk);
+            assertRunsWithJavaHome();
+        } finally {
+            mv(relocatedJdk, installation.bundledJdk);
+        }
+    }
+
     public void test50Remove() throws Exception {
         // add fake bin directory as if a plugin was installed
         Files.createDirectories(installation.bin.resolve("myplugin"));
@@ -300,12 +295,12 @@ public class PackageTests extends PackagingTestCase {
         stopElasticsearch();
 
         withCustomConfig(tempConf -> {
-            append(installation.envFile, "ES_JAVA_OPTS=-XX:-UseCompressedOops");
+            append(installation.envFile, "ES_JAVA_OPTS=\"-Xmx512m -Xms512m -XX:-UseCompressedOops\"");
 
             startElasticsearch();
 
             final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes"));
-            assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":1073741824"));
+            assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":536870912"));
             assertThat(nodesResponse, containsString("\"using_compressed_ordinary_object_pointers\":\"false\""));
 
             stopElasticsearch();

qa/os/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java

@@ -59,10 +59,12 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.StandardCopyOption;
+import java.nio.file.StandardOpenOption;
 import java.nio.file.attribute.FileAttribute;
 import java.nio.file.attribute.PosixFilePermissions;
 import java.util.Collections;
 import java.util.List;
+import java.util.Locale;
 
 import static org.elasticsearch.packaging.util.Cleanup.cleanEverything;
 import static org.elasticsearch.packaging.util.Docker.ensureImageIsLoaded;
@@ -173,6 +175,9 @@ public abstract class PackagingTestCase extends Assert {
             Platforms.onLinux(() -> sh.getEnv().put("JAVA_HOME", systemJavaHome));
             Platforms.onWindows(() -> sh.getEnv().put("JAVA_HOME", systemJavaHome));
         }
+        if (installation != null && distribution.isDocker() == false) {
+            setHeap("1g");
+        }
     }
 
     @After
@@ -224,6 +229,11 @@ public abstract class PackagingTestCase extends Assert {
             default:
                 throw new IllegalStateException("Unknown Elasticsearch packaging type.");
         }
+
+        // the purpose of the packaging tests are not to all test auto heap, so we explicitly set heap size to 1g
+        if (distribution.isDocker() == false) {
+            setHeap("1g");
+        }
     }
 
     protected static void cleanup() throws Exception {
@@ -447,4 +457,25 @@ public abstract class PackagingTestCase extends Assert {
         }
         IOUtils.rm(tempDir);
     }
+
+    /**
+     * Manually set the heap size with a jvm.options.d file. This will be reset before each test.
+     */
+    public static void setHeap(String heapSize) throws IOException {
+        setHeap(heapSize, installation.config);
+    }
+
+    public static void setHeap(String heapSize, Path config) throws IOException {
+        Path heapOptions = config.resolve("jvm.options.d").resolve("heap.options");
+        if (heapSize == null) {
+            FileUtils.rm(heapOptions);
+        } else {
+            Files.writeString(
+                heapOptions,
+                String.format(Locale.ROOT, "-Xmx%1$s%n-Xms%1$s%n", heapSize),
+                StandardOpenOption.CREATE,
+                StandardOpenOption.TRUNCATE_EXISTING
+            );
+        }
+    }
 }

qa/os/src/test/java/org/elasticsearch/packaging/test/RpmPreservationTests.java

@@ -57,6 +57,7 @@ public class RpmPreservationTests extends PackagingTestCase {
     }
 
     public void test20Remove() throws Exception {
+        setHeap(null); // remove test heap options, so the config directory can be removed
         remove(distribution());
 
         // config was removed
@@ -64,6 +65,9 @@ public class RpmPreservationTests extends PackagingTestCase {
 
         // defaults file was removed
         assertThat(installation.envFile, fileDoesNotExist());
+
+        // don't perform normal setup/teardown after this since we removed the install
+        installation = null;
     }
 
     public void test30PreserveConfig() throws Exception {

qa/os/src/test/java/org/elasticsearch/packaging/util/DockerRun.java

@@ -38,6 +38,7 @@ public class DockerRun {
     private Integer uid;
     private Integer gid;
     private final List<String> extraArgs = new ArrayList<>();
+    private String memory = "2g"; // default to 2g memory limit
 
     private DockerRun() {}
 
@@ -75,6 +76,13 @@ public class DockerRun {
         return this;
     }
 
+    public DockerRun memory(String memoryLimit) {
+        if (memoryLimit != null) {
+            this.memory = memoryLimit;
+        }
+        return this;
+    }
+
     public DockerRun extraArgs(String... args) {
         Collections.addAll(this.extraArgs, args);
         return this;
@@ -88,6 +96,9 @@ public class DockerRun {
         // Run the container in the background
         cmd.add("--detach");
 
+        // Limit container memory
+        cmd.add("--memory " + memory);
+
         this.envVars.forEach((key, value) -> cmd.add("--env " + key + "=\"" + value + "\""));
 
         // The container won't run without configuring discovery