Browse Source

Introduce jvm.options.d for customizing JVM options (#51882)

This commit introduces the ability to override JVM options by adding
custom JVM options files to a jvm.options.d directory. This simplifies
administration of Elasticsearch by not requiring administrators to keep
the root jvm.options file in sync with changes that we make to the root
jvm.options file. Instead, they are not expected to modify this file but
instead supply their own in jvm.options.d. In Docker installations, this
means they can bind mount this directory in. In future versions of
Elasticsearch, we can consider removing the root jvm.options file
(instead, providing all options there as system JVM options).
Jason Tedor 5 years ago
parent
commit
749b623de0

+ 10 - 1
distribution/archives/build.gradle

@@ -44,6 +44,11 @@ task createPluginsDir(type: EmptyDirTask) {
   dir = "${pluginsDir}"
   dirMode = 0755
 }
+ext.jvmOptionsDir = new File(buildDir, 'jvm-options-hack/jvm.options.d')
+task createJvmOptionsDir(type: EmptyDirTask) {
+  dir = "${jvmOptionsDir}"
+  dirMode = 0750
+}
 
 CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String platform, boolean oss, boolean jdk) {
   return copySpec {
@@ -55,6 +60,10 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla
         dirMode 0750
         fileMode 0660
         with configFiles(distributionType, oss, jdk)
+        from {
+          dirMode 0750
+          jvmOptionsDir.getParent()
+        }
       }
       into('bin') {
         with binFiles(distributionType, oss, jdk)
@@ -94,7 +103,7 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla
 
 // common config across all zip/tar
 tasks.withType(AbstractArchiveTask) {
-  dependsOn createLogsDir, createPluginsDir
+  dependsOn createLogsDir, createPluginsDir, createJvmOptionsDir
   String subdir = it.name.substring('build'.size()).replaceAll(/[A-Z]/) { '-' + it.toLowerCase() }.substring(1)
   destinationDir = file("${subdir}/build/distributions")
   baseName = "elasticsearch${subdir.contains('oss') ? '-oss' : ''}"

+ 2 - 2
distribution/docker/src/docker/Dockerfile

@@ -30,8 +30,8 @@ ${source_elasticsearch}
 RUN tar zxf /opt/${elasticsearch} --strip-components=1
 RUN grep ES_DISTRIBUTION_TYPE=tar /usr/share/elasticsearch/bin/elasticsearch-env \
     && sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' /usr/share/elasticsearch/bin/elasticsearch-env
-RUN mkdir -p config data logs
-RUN chmod 0775 config data logs
+RUN mkdir -p config config/jvm.options.d data logs
+RUN chmod 0775 config config/jvm.options.d data logs
 COPY config/elasticsearch.yml config/log4j2.properties config/
 RUN chmod 0660 config/elasticsearch.yml config/log4j2.properties
 

+ 3 - 1
distribution/packages/build.gradle

@@ -79,8 +79,9 @@ void addProcessFilesTask(String type, boolean oss, boolean jdk) {
       mkdir "${packagingFiles}/var/lib/elasticsearch"
       mkdir "${packagingFiles}/usr/share/elasticsearch/plugins"
 
-      // bare empty dir for /etc/elasticsearch
+      // bare empty dir for /etc/elasticsearch and /etc/elasticsearch/jvm.options.d
       mkdir "${packagingFiles}/elasticsearch"
+      mkdir "${packagingFiles}/elasticsearch/jvm.options.d"
     }
   }
 }
@@ -197,6 +198,7 @@ Closure commonPackageConfig(String type, boolean oss, boolean jdk) {
       includeEmptyDirs true
       createDirectoryEntry true
       include("elasticsearch") // empty dir, just to add directory entry
+      include("elasticsearch/jvm.options.d") // empty dir, just to add directory entry
     }
     from("${packagingFiles}/etc/elasticsearch") {
       into('/etc/elasticsearch')

+ 17 - 0
distribution/packages/src/common/scripts/postrm

@@ -17,6 +17,7 @@ else
 fi
 
 REMOVE_DIRS=false
+REMOVE_JVM_OPTIONS_DIRECTORY=false
 REMOVE_USER_AND_GROUP=false
 
 case "$1" in
@@ -27,6 +28,8 @@ case "$1" in
     ;;
 
     purge)
+        REMOVE_DIRS=true
+        REMOVE_JVM_OPTIONS_DIRECTORY=true
         REMOVE_USER_AND_GROUP=true
     ;;
     failed-upgrade|abort-install|abort-upgrade|disappear|upgrade|disappear)
@@ -80,6 +83,20 @@ if [ "$REMOVE_DIRS" = "true" ]; then
         rmdir --ignore-fail-on-non-empty /var/lib/elasticsearch
     fi
 
+    # delete the jvm.options.d directory if and only if empty
+    if [ -d "${ES_PATH_CONF}/jvm.options.d" ]; then
+        rmdir --ignore-fail-on-non-empty "${ES_PATH_CONF}/jvm.options.d"
+    fi
+
+    # delete the jvm.options.d directory if we are purging
+    if [ "$REMOVE_JVM_OPTIONS_DIRECTORY" = "true" ]; then
+      if [ -d "${ES_PATH_CONF}/jvm.options.d" ]; then
+        echo -n "Deleting jvm.options.d directory..."
+        rm -rf "${ES_PATH_CONF}/jvm.options.d"
+        echo " OK"
+      fi
+    fi
+
     # delete the conf directory if and only if empty
     if [ -d "${ES_PATH_CONF}" ]; then
         rmdir --ignore-fail-on-non-empty "${ES_PATH_CONF}"

+ 1 - 0
distribution/packages/src/deb/lintian/elasticsearch

@@ -16,6 +16,7 @@ missing-dep-on-jarwrapper
 # we prefer to not make our config and log files world readable
 non-standard-file-perm etc/default/elasticsearch 0660 != 0644
 non-standard-dir-perm etc/elasticsearch/ 2750 != 0755
+non-standard-dir-perm etc/elasticsearch/jvm.options.d/ 2750 != 0755
 non-standard-file-perm etc/elasticsearch/*
 non-standard-dir-perm var/lib/elasticsearch/ 2750 != 0755
 non-standard-dir-perm var/log/elasticsearch/ 2750 != 0755

+ 1 - 2
distribution/src/bin/elasticsearch

@@ -46,8 +46,7 @@ then
   fi
 fi
 
-ES_JVM_OPTIONS="$ES_PATH_CONF"/jvm.options
-ES_JAVA_OPTS=`export ES_TMPDIR; "$JAVA" -cp "$ES_CLASSPATH" org.elasticsearch.tools.launchers.JvmOptionsParser "$ES_JVM_OPTIONS"`
+ES_JAVA_OPTS=`export ES_TMPDIR; "$JAVA" -cp "$ES_CLASSPATH" org.elasticsearch.tools.launchers.JvmOptionsParser "$ES_PATH_CONF"`
 
 # manual parsing to find out, if process should be detached
 if [[ $DAEMONIZE = false ]]; then

+ 4 - 4
distribution/src/bin/elasticsearch-service.bat

@@ -115,7 +115,7 @@ set ES_JVM_OPTIONS=%ES_PATH_CONF%\jvm.options
 if not "%ES_JAVA_OPTS%" == "" set ES_JAVA_OPTS=%ES_JAVA_OPTS: =;%
 
 @setlocal
-for /F "usebackq delims=" %%a in (`"%JAVA% -cp "!ES_CLASSPATH!" "org.elasticsearch.tools.launchers.JvmOptionsParser" "!ES_JVM_OPTIONS!" || echo jvm_options_parser_failed"`) do set ES_JAVA_OPTS=%%a
+for /F "usebackq delims=" %%a in (`"%JAVA% -cp "!ES_CLASSPATH!" "org.elasticsearch.tools.launchers.JvmOptionsParser" "!ES_PATH_CONF!" || echo jvm_options_parser_failed"`) do set ES_JAVA_OPTS=%%a
 @endlocal & set "MAYBE_JVM_OPTIONS_PARSER_FAILED=%ES_JAVA_OPTS%" & set ES_JAVA_OPTS=%ES_JAVA_OPTS%
 
 if "%MAYBE_JVM_OPTIONS_PARSER_FAILED%" == "jvm_options_parser_failed" (
@@ -167,15 +167,15 @@ for %%a in ("%ES_JAVA_OPTS:;=","%") do (
 @endlocal & set JVM_MS=%JVM_MS% & set JVM_MX=%JVM_MX% & set JVM_SS=%JVM_SS% & set OTHER_JAVA_OPTS=%OTHER_JAVA_OPTS%
 
 if "%JVM_MS%" == "" (
-  echo minimum heap size not set; configure using -Xms via "%ES_JVM_OPTIONS%" or ES_JAVA_OPTS
+  echo minimum heap size not set; configure using -Xms via "%ES_PATH_CONF%/jvm.options.d", or ES_JAVA_OPTS
   goto:eof
 )
 if "%JVM_MX%" == "" (
-  echo maximum heap size not set; configure using -Xmx via "%ES_JVM_OPTIONS%" or ES_JAVA_OPTS
+  echo maximum heap size not set; configure using -Xmx via "%ES_PATH_CONF%/jvm.options.d", or ES_JAVA_OPTS
   goto:eof
 )
 if "%JVM_SS%" == "" (
-  echo thread stack size not set; configure using -Xss via "%ES_JVM_OPTIONS%" or ES_JAVA_OPTS
+  echo thread stack size not set; configure using -Xss via "%ES_PATH_CONF%/jvm.options.d", or ES_JAVA_OPTS
   goto:eof
 )
 set OTHER_JAVA_OPTS=%OTHER_JAVA_OPTS:"=%

+ 1 - 2
distribution/src/bin/elasticsearch.bat

@@ -72,9 +72,8 @@ if not defined ES_TMPDIR (
   for /f "tokens=* usebackq" %%a in (`CALL %JAVA% -cp "!ES_CLASSPATH!" "org.elasticsearch.tools.launchers.TempDirectory"`) do set  ES_TMPDIR=%%a
 )
 
-set ES_JVM_OPTIONS=%ES_PATH_CONF%\jvm.options
 @setlocal
-for /F "usebackq delims=" %%a in (`CALL %JAVA% -cp "!ES_CLASSPATH!" "org.elasticsearch.tools.launchers.JvmOptionsParser" "!ES_JVM_OPTIONS!" ^|^| echo jvm_options_parser_failed`) do set ES_JAVA_OPTS=%%a
+for /F "usebackq delims=" %%a in (`CALL %JAVA% -cp "!ES_CLASSPATH!" "org.elasticsearch.tools.launchers.JvmOptionsParser" "!ES_PATH_CONF!" ^|^| echo jvm_options_parser_failed`) do set ES_JAVA_OPTS=%%a
 @endlocal & set "MAYBE_JVM_OPTIONS_PARSER_FAILED=%ES_JAVA_OPTS%" & set ES_JAVA_OPTS=%ES_JAVA_OPTS%
 
 if "%MAYBE_JVM_OPTIONS_PARSER_FAILED%" == "jvm_options_parser_failed" (

+ 76 - 65
distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOptionsParser.java

@@ -27,7 +27,9 @@ import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.Reader;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
+import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -43,6 +45,7 @@ import java.util.function.Predicate;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
 
 /**
  * Parses JVM options from a file and prints a single line with all JVM options to standard output.
@@ -53,82 +56,90 @@ final class JvmOptionsParser {
      * The main entry point. The exit code is 0 if the JVM options were successfully parsed, otherwise the exit code is 1. If an improperly
      * formatted line is discovered, the line is output to standard error.
      *
-     * @param args the args to the program which should consist of a single option, the path to the JVM options
+     * @param args the args to the program which should consist of a single option, the path to ES_PATH_CONF
      */
     public static void main(final String[] args) throws InterruptedException, IOException {
         if (args.length != 1) {
-            throw new IllegalArgumentException("expected one argument specifying path to jvm.options but was " + Arrays.toString(args));
-        }
-        final List<String> jvmOptions = new ArrayList<>();
-        final SortedMap<Integer, String> invalidLines = new TreeMap<>();
-        try (
-            InputStream is = Files.newInputStream(Paths.get(args[0]));
-            Reader reader = new InputStreamReader(is, StandardCharsets.UTF_8);
-            BufferedReader br = new BufferedReader(reader)
-        ) {
-            parse(JavaVersion.majorVersion(JavaVersion.CURRENT), br, new JvmOptionConsumer() {
-                @Override
-                public void accept(final String jvmOption) {
-                    jvmOptions.add(jvmOption);
-                }
-            }, new InvalidLineConsumer() {
-                @Override
-                public void accept(final int lineNumber, final String line) {
-                    invalidLines.put(lineNumber, line);
-                }
-            });
+            throw new IllegalArgumentException("expected one argument specifying path to ES_PATH_CONF but was " + Arrays.toString(args));
         }
 
-        if (invalidLines.isEmpty()) {
-            // now append the JVM options from ES_JAVA_OPTS
-            final String environmentJvmOptions = System.getenv("ES_JAVA_OPTS");
-            if (environmentJvmOptions != null) {
-                jvmOptions.addAll(
-                    Arrays.stream(environmentJvmOptions.split("\\s+"))
-                        .filter(Predicate.not(String::isBlank))
-                        .collect(Collectors.toUnmodifiableList())
-                );
+        final ArrayList<Path> jvmOptionsFiles = new ArrayList<>();
+        jvmOptionsFiles.add(Paths.get(args[0], "jvm.options"));
+
+        final Path jvmOptionsDirectory = Paths.get(args[0], "jvm.options.d");
+
+        if (Files.isDirectory(jvmOptionsDirectory)) {
+            try (
+                DirectoryStream<Path> jvmOptionsDirectoryStream = Files.newDirectoryStream(Paths.get(args[0], "jvm.options.d"), "*.options")
+            ) {
+                // collect the matching JVM options files after sorting them by Path::compareTo
+                StreamSupport.stream(jvmOptionsDirectoryStream.spliterator(), false).sorted().forEach(jvmOptionsFiles::add);
             }
-            final Map<String, String> substitutions = new HashMap<>();
-            substitutions.put("ES_TMPDIR", System.getenv("ES_TMPDIR"));
-            if (null != System.getenv("ES_PATH_CONF")) {
-                substitutions.put("ES_PATH_CONF", System.getenv("ES_PATH_CONF"));
+        }
+
+        final List<String> jvmOptions = new ArrayList<>();
+
+        for (final Path jvmOptionsFile : jvmOptionsFiles) {
+            final SortedMap<Integer, String> invalidLines = new TreeMap<>();
+            try (
+                InputStream is = Files.newInputStream(jvmOptionsFile);
+                Reader reader = new InputStreamReader(is, StandardCharsets.UTF_8);
+                BufferedReader br = new BufferedReader(reader)
+            ) {
+                parse(JavaVersion.majorVersion(JavaVersion.CURRENT), br, jvmOptions::add, invalidLines::put);
             }
-            final List<String> substitutedJvmOptions = substitutePlaceholders(jvmOptions, Collections.unmodifiableMap(substitutions));
-            final List<String> ergonomicJvmOptions = JvmErgonomics.choose(substitutedJvmOptions);
-            final List<String> systemJvmOptions = SystemJvmOptions.systemJvmOptions();
-            final List<String> finalJvmOptions = new ArrayList<>(
-                systemJvmOptions.size() + substitutedJvmOptions.size() + ergonomicJvmOptions.size()
-            );
-            finalJvmOptions.addAll(systemJvmOptions); // add the system JVM options first so that they can be overridden
-            finalJvmOptions.addAll(substitutedJvmOptions);
-            finalJvmOptions.addAll(ergonomicJvmOptions);
-            final String spaceDelimitedJvmOptions = spaceDelimitJvmOptions(finalJvmOptions);
-            Launchers.outPrintln(spaceDelimitedJvmOptions);
-            Launchers.exit(0);
-        } else {
-            final String errorMessage = String.format(
-                Locale.ROOT,
-                "encountered [%d] error%s parsing [%s]",
-                invalidLines.size(),
-                invalidLines.size() == 1 ? "" : "s",
-                args[0]
-            );
-            Launchers.errPrintln(errorMessage);
-            int count = 0;
-            for (final Map.Entry<Integer, String> entry : invalidLines.entrySet()) {
-                count++;
-                final String message = String.format(
+            if (invalidLines.isEmpty() == false) {
+                final String errorMessage = String.format(
                     Locale.ROOT,
-                    "[%d]: encountered improperly formatted JVM option line [%s] on line number [%d]",
-                    count,
-                    entry.getValue(),
-                    entry.getKey()
+                    "encountered [%d] error%s parsing [%s]",
+                    invalidLines.size(),
+                    invalidLines.size() == 1 ? "" : "s",
+                    jvmOptionsFile
                 );
-                Launchers.errPrintln(message);
+                Launchers.errPrintln(errorMessage);
+                int count = 0;
+                for (final Map.Entry<Integer, String> entry : invalidLines.entrySet()) {
+                    count++;
+                    final String message = String.format(
+                        Locale.ROOT,
+                        "[%d]: encountered improperly formatted JVM option in [%s] on line number [%d]: [%s]",
+                        count,
+                        jvmOptionsFile,
+                        entry.getKey(),
+                        entry.getValue()
+                    );
+                    Launchers.errPrintln(message);
+                }
+                Launchers.exit(1);
             }
-            Launchers.exit(1);
         }
+
+        // now append the JVM options from ES_JAVA_OPTS
+        final String environmentJvmOptions = System.getenv("ES_JAVA_OPTS");
+        if (environmentJvmOptions != null) {
+            jvmOptions.addAll(
+                Arrays.stream(environmentJvmOptions.split("\\s+"))
+                    .filter(Predicate.not(String::isBlank))
+                    .collect(Collectors.toUnmodifiableList())
+            );
+        }
+        final Map<String, String> substitutions = new HashMap<>();
+        substitutions.put("ES_TMPDIR", System.getenv("ES_TMPDIR"));
+        if (null != System.getenv("ES_PATH_CONF")) {
+            substitutions.put("ES_PATH_CONF", System.getenv("ES_PATH_CONF"));
+        }
+        final List<String> substitutedJvmOptions = substitutePlaceholders(jvmOptions, Collections.unmodifiableMap(substitutions));
+        final List<String> ergonomicJvmOptions = JvmErgonomics.choose(substitutedJvmOptions);
+        final List<String> systemJvmOptions = SystemJvmOptions.systemJvmOptions();
+        final List<String> finalJvmOptions = new ArrayList<>(
+            systemJvmOptions.size() + substitutedJvmOptions.size() + ergonomicJvmOptions.size()
+        );
+        finalJvmOptions.addAll(systemJvmOptions); // add the system JVM options first so that they can be overridden
+        finalJvmOptions.addAll(substitutedJvmOptions);
+        finalJvmOptions.addAll(ergonomicJvmOptions);
+        final String spaceDelimitedJvmOptions = spaceDelimitJvmOptions(finalJvmOptions);
+        Launchers.outPrintln(spaceDelimitedJvmOptions);
+        Launchers.exit(0);
     }
 
     static List<String> substitutePlaceholders(final List<String> jvmOptions, final Map<String, String> substitutions) {

+ 2 - 3
docs/reference/setup/important-settings/heap-size.asciidoc

@@ -49,7 +49,7 @@ caches, but the less memory it leaves available for the operating system to use
 for the filesystem cache. Also, larger heaps can cause longer garbage
 collection pauses.
 
-Here are examples of how to set the heap size via the jvm.options file:
+Here is an example of how to set the heap size via a `jvm.options.d/` file:
 
 [source,txt]
 ------------------
@@ -60,8 +60,7 @@ Here are examples of how to set the heap size via the jvm.options file:
 <2> Set the maximum heap size to 2g.
 
 It is also possible to set the heap size via an environment variable. This can
-be done by commenting out the `Xms` and `Xmx` settings in the
-<<jvm-options,`jvm.options`>> file and setting these values via `ES_JAVA_OPTS`:
+be done by setting these values via `ES_JAVA_OPTS`:
 
 [source,sh]
 ------------------

+ 12 - 11
docs/reference/setup/install/docker.asciidoc

@@ -266,22 +266,23 @@ unless you are pinning one container per host.
 [[docker-set-heap-size]]
 ===== Set the heap size
 
-Use the `ES_JAVA_OPTS` environment variable to set the heap size.
-For example, to use 16GB, specify `-e ES_JAVA_OPTS="-Xms16g -Xmx16g"` with
-`docker run`. Note that while the default configuration file `jvm.options`
-sets a default heap of 1GB, any value you set in `ES_JAVA_OPTS` will
-override it.
+To configure the heap size, you can bind mount a <<jvm-options,JVM options>>
+file under `/usr/share/elasticsearch/config/jvm.options.d` that includes your
+desired <<heap-size,heap size>> settings. Note that while the default root
+`jvm.options` file sets a default heap of 1 GB, any value you set in a
+bind-mounted JVM options file will override it.
+
+While setting the heap size via bind-mounted JVM options is the recommended
+method, you can also configure this by using the `ES_JAVA_OPTS` environment
+variable to set the heap size. For example, to use 16 GB, specify
+`-e ES_JAVA_OPTS="-Xms16g -Xmx16g"` with `docker run`. Note that while the
+default root `jvm.options` file sets a default heap of 1 GB, any value you set
+in `ES_JAVA_OPTS` will override it.
 
 IMPORTANT: You must <<heap-size,configure the heap size>> even if you are
 https://docs.docker.com/config/containers/resource_constraints/#limit-a-containers-access-to-memory[limiting
 memory access] to the container.
 
-While setting the heap size via an environment variable is the recommended
-method, you can also configure this by bind-mounting your own `jvm.options`
-file under `/usr/share/elasticsearch/config/`. The file that {es} provides
-contains some important settings, so you should start by taking a copy of
-`jvm.options` from an {es} container and editing it as you require.
-
 ===== Pin deployments to a specific image version
 
 Pin your deployments to a specific version of the {es} Docker image. For

+ 15 - 16
docs/reference/setup/jvm-options.asciidoc

@@ -5,18 +5,20 @@ You should rarely need to change Java Virtual Machine (JVM) options. If you do,
 the most likely change is setting the <<heap-size,heap size>>. The remainder of
 this document explains in detail how to set JVM options.
 
-The preferred method of setting JVM options (including system properties and JVM
-flags) is via the `jvm.options` configuration file. The default location of this
-file is `config/jvm.options` (when installing from the tar or zip distributions)
-and `/etc/elasticsearch/jvm.options` (when installing from the Debian or RPM
-packages).
-
-NOTE: If you are using the <<docker,Docker distribution of {es}>>, we
-recommend that you <<docker-set-heap-size,set the heap size using the
-`ES_JAVA_OPTS` environment variable>>.
-
-This file contains a line-delimited list of JVM arguments following
-a special syntax:
+The preferred method of setting or overriding JVM options is via JVM options
+files. When installing from the tar or zip distributions, the root `jvm.options`
+configuration file is `config/jvm.options` and custom JVM options files can be
+added to `config/jvm.options.d/`. When installing from the Debian or RPM
+packages, the root `jvm.options` configuration file is
+``/etc/elasticsearch/jvm.options` and custom JVM options files can be added to
+`/etc/elasticsearch/jvm.options.d/`. When using the <<docker, Docker
+distribution of {es}>> you can bind mount custom JVM options files into
+`/usr/share/elasticsearch/config/jvm.options.d/`. You should never need to
+modify the root `jvm.options` file instead preferring to use custom JVM options
+files. The processing ordering of custom JVM options is lexicographic.
+
+JVM options files must have the suffix '.options' and contain a line-delimited
+list of JVM arguments following a special syntax:
 
 * lines consisting of whitespace only are ignored
 * lines beginning with `#` are treated as comments and are ignored
@@ -62,9 +64,6 @@ a special syntax:
 
 * all other lines are rejected
 
-You can add custom JVM flags to this file and check this configuration into your
-version control system.
-
 An alternative mechanism for setting Java Virtual Machine options is via the
 `ES_JAVA_OPTS` environment variable. For instance:
 
@@ -81,7 +80,7 @@ The JVM has a built-in mechanism for observing the `JAVA_TOOL_OPTIONS`
 environment variable. We intentionally ignore this environment variable in our
 packaging scripts. The primary reason for this is that on some OS (e.g., Ubuntu)
 there are agents installed by default via this environment variable that we do
-not want interfering with Elasticsearch.
+not want interfering with {es}.
 
 Additionally, some other Java programs support the `JAVA_OPTS` environment
 variable. This is *not* a mechanism built into the JVM but instead a convention

+ 56 - 0
qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java

@@ -253,6 +253,62 @@ public class ArchiveTests extends PackagingTestCase {
         }
     }
 
+    public void test71CustomJvmOptionsDirectoryFile() throws Exception {
+        final Path heapOptions = installation.config(Paths.get("jvm.options.d", "heap.options"));
+        try {
+            append(heapOptions, "-Xms512m\n-Xmx512m\n");
+
+            startElasticsearch();
+
+            final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes"));
+            assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":536870912"));
+
+            stopElasticsearch();
+        } finally {
+            rm(heapOptions);
+        }
+    }
+
+    public void test72CustomJvmOptionsDirectoryFilesAreProcessedInSortedOrder() throws Exception {
+        final Path firstOptions = installation.config(Paths.get("jvm.options.d", "first.options"));
+        final Path secondOptions = installation.config(Paths.get("jvm.options.d", "second.options"));
+        try {
+            /*
+             * We override the heap in the first file, and disable compressed oops, and override the heap in the second file. By doing this,
+             * we can test that both files are processed by the JVM options parser, and also that they are processed in lexicographic order.
+             */
+            append(firstOptions, "-Xms384m\n-Xmx384m\n-XX:-UseCompressedOops\n");
+            append(secondOptions, "-Xms512m\n-Xmx512m\n");
+
+            startElasticsearch();
+
+            final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes"));
+            assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":536870912"));
+            assertThat(nodesResponse, containsString("\"using_compressed_ordinary_object_pointers\":\"false\""));
+
+            stopElasticsearch();
+        } finally {
+            rm(firstOptions);
+            rm(secondOptions);
+        }
+    }
+
+    public void test73CustomJvmOptionsDirectoryFilesWithoutOptionsExtensionIgnored() throws Exception {
+        final Path jvmOptionsIgnored = installation.config(Paths.get("jvm.options.d", "jvm.options.ignored"));
+        try {
+            append(jvmOptionsIgnored, "-Xms512\n-Xmx512m\n");
+
+            startElasticsearch();
+
+            final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes"));
+            assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":1073741824"));
+
+            stopElasticsearch();
+        } finally {
+            rm(jvmOptionsIgnored);
+        }
+    }
+
     public void test80RelativePathConf() throws Exception {
 
         final Path temp = getTempDir().resolve("esconf-alternate");

+ 7 - 2
qa/os/src/test/java/org/elasticsearch/packaging/test/DebPreservationTests.java

@@ -25,6 +25,7 @@ import org.junit.BeforeClass;
 import java.nio.file.Paths;
 
 import static org.elasticsearch.packaging.util.FileExistenceMatchers.fileExists;
+import static org.elasticsearch.packaging.util.FileUtils.append;
 import static org.elasticsearch.packaging.util.FileUtils.assertPathsDoNotExist;
 import static org.elasticsearch.packaging.util.FileUtils.assertPathsExist;
 import static org.elasticsearch.packaging.util.Packages.SYSVINIT_SCRIPT;
@@ -53,15 +54,17 @@ public class DebPreservationTests extends PackagingTestCase {
     }
 
     public void test20Remove() throws Exception {
+        append(installation.config(Paths.get("jvm.options.d", "heap.options")), "# foo");
+
         remove(distribution());
 
         // some config files were not removed
-
         assertPathsExist(
             installation.config,
             installation.config("elasticsearch.yml"),
             installation.config("jvm.options"),
-            installation.config("log4j2.properties")
+            installation.config("log4j2.properties"),
+            installation.config(Paths.get("jvm.options.d", "heap.options"))
         );
 
         if (distribution().isDefault()) {
@@ -96,6 +99,8 @@ public class DebPreservationTests extends PackagingTestCase {
     }
 
     public void test30Purge() throws Exception {
+        append(installation.config(Paths.get("jvm.options.d", "heap.options")), "# foo");
+
         sh.run("dpkg --purge " + distribution().flavor.name);
 
         assertRemoved(distribution());

+ 18 - 2
qa/os/src/test/java/org/elasticsearch/packaging/test/PackageTests.java

@@ -138,6 +138,22 @@ public class PackageTests extends PackagingTestCase {
         }
     }
 
+    public void test34CustomJvmOptionsDirectoryFile() throws Exception {
+        final Path heapOptions = installation.config(Paths.get("jvm.options.d", "heap.options"));
+        try {
+            append(heapOptions, "-Xms512m\n-Xmx512m\n");
+
+            startElasticsearch();
+
+            final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes"));
+            assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":536870912"));
+
+            stopElasticsearch();
+        } finally {
+            rm(heapOptions);
+        }
+    }
+
     public void test42BundledJdkRemoved() throws Exception {
         assumeThat(distribution().hasJdk, is(true));
 
@@ -300,7 +316,7 @@ public class PackageTests extends PackagingTestCase {
         });
     }
 
-    public void test82SystemdMask() throws Exception {
+    public void test83SystemdMask() throws Exception {
         try {
             assumeTrue(isSystemd());
 
@@ -313,7 +329,7 @@ public class PackageTests extends PackagingTestCase {
         }
     }
 
-    public void test83serviceFileSetsLimits() throws Exception {
+    public void test84serviceFileSetsLimits() throws Exception {
         // Limits are changed on systemd platforms only
         assumeTrue(isSystemd());
 

+ 8 - 0
qa/os/src/test/java/org/elasticsearch/packaging/test/RpmPreservationTests.java

@@ -24,6 +24,7 @@ import org.elasticsearch.packaging.util.Shell;
 import org.junit.BeforeClass;
 
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.stream.Stream;
 
 import static org.elasticsearch.packaging.util.FileExistenceMatchers.fileDoesNotExist;
@@ -84,6 +85,7 @@ public class RpmPreservationTests extends PackagingTestCase {
         )
             .map(each -> installation.config(each))
             .forEach(path -> append(path, "# foo"));
+        append(installation.config(Paths.get("jvm.options.d", "heap.options")), "# foo");
         if (distribution().isDefault()) {
             Stream.of(
                 "role_mapping.yml",
@@ -122,6 +124,7 @@ public class RpmPreservationTests extends PackagingTestCase {
             "jvm.options",
             "log4j2.properties"
         ).forEach(this::assertConfFilePreserved);
+        assertThat(installation.config(Paths.get("jvm.options.d", "heap.options")), fileExists());
 
         if (distribution().isDefault()) {
             Stream.of(
@@ -136,7 +139,12 @@ public class RpmPreservationTests extends PackagingTestCase {
     private void assertConfFilePreserved(String configFile) {
         final Path original = installation.config(configFile);
         final Path saved = installation.config(configFile + ".rpmsave");
+        assertConfFilePreserved(original ,saved);
+    }
+
+    private void assertConfFilePreserved(final Path original, final Path saved) {
         assertThat(original, fileDoesNotExist());
         assertThat(saved, fileExists());
     }
+
 }

+ 4 - 0
qa/os/src/test/java/org/elasticsearch/packaging/util/Installation.java

@@ -135,6 +135,10 @@ public class Installation {
         return config.resolve(configFileName);
     }
 
+    public Path config(Path configFileName) {
+        return config.resolve(configFileName);
+    }
+
     public Executables executables() {
         return new Executables();
     }

+ 4 - 0
qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java

@@ -182,6 +182,10 @@ public class Packages {
         assertThat(es.config, file(Directory, "root", "elasticsearch", p750));
         assertThat(sh.run("find \"" + es.config + "\" -maxdepth 0 -printf \"%m\"").stdout, containsString("2750"));
 
+        final Path jvmOptionsDirectory = es.config.resolve("jvm.options.d");
+        assertThat(jvmOptionsDirectory, file(Directory, "root", "elasticsearch", p750));
+        assertThat(sh.run("find \"" + jvmOptionsDirectory + "\" -maxdepth 0 -printf \"%m\"").stdout, containsString("2750"));
+
         Stream.of(
             "elasticsearch.keystore",
             "elasticsearch.yml",