Support `_FILE` suffixed env vars in Docker entrypoint (#47573)

Closes #43603. Allow environment variables to be passed to ES in a Docker
container via a file, by setting an environment variable with the `_FILE`
suffix that points to the file with the intended value of the env var.

distribution/docker/src/docker/bin/docker-entrypoint.sh

@@ -38,6 +38,40 @@ if [[ "$1" != "eswrapper" ]]; then
   fi
 fi
 
+# Allow environment variables to be set by creating a file with the
+# contents, and setting an environment variable with the suffix _FILE to
+# point to it. This can be used to provide secrets to a container, without
+# the values being specified explicitly when running the container.
+for VAR_NAME_FILE in $(env | cut -f1 -d= | grep '_FILE$'); do
+  if [[ -n "$VAR_NAME_FILE" ]]; then
+    VAR_NAME="${VAR_NAME_FILE%_FILE}"
+
+    if env | grep "^${VAR_NAME}="; then
+      echo "ERROR: Both $VAR_NAME_FILE and $VAR_NAME are set. These are mutually exclusive." >&2
+      exit 1
+    fi
+
+    if [[ ! -e "${!VAR_NAME_FILE}" ]]; then
+      echo "ERROR: File ${!VAR_NAME_FILE} from $VAR_NAME_FILE does not exist" >&2
+      exit 1
+    fi
+
+    FILE_PERMS="$(stat -c '%a' ${!VAR_NAME_FILE})"
+
+    if [[ "$FILE_PERMS" != "400" && "$FILE_PERMS" != 600 ]]; then
+        echo "ERROR: File ${!VAR_NAME_FILE} from $VAR_NAME_FILE must have file permissions 400 or 600, but actually has: $FILE_PERMS" >&2
+        exit 1
+    fi
+
+    echo "Setting $VAR_NAME from $VAR_NAME_FILE at ${!VAR_NAME_FILE}" >&2
+    export "$VAR_NAME"="$(cat ${!VAR_NAME_FILE})"
+
+    unset VAR_NAME
+    # Unset the suffixed environment variable
+    unset "$VAR_NAME_FILE"
+  fi
+done
+
 # Parse Docker env vars to customize Elasticsearch
 #
 # e.g. Setting the env var cluster.name=testcluster

docs/reference/setup/install/docker.asciidoc

@@ -290,7 +290,7 @@ https://docs.docker.com/engine/extend/plugins/#volume-plugins[Docker volume plug
 
 ===== Avoid using `loop-lvm` mode
 
-If you are using the devicemapper storage driver, do not use the default `loop-lvm` mode. 
+If you are using the devicemapper storage driver, do not use the default `loop-lvm` mode.
 Configure docker-engine to use 
 https://docs.docker.com/engine/userguide/storagedriver/device-mapper-driver/#configure-docker-with-devicemapper[direct-lvm].
 
@@ -312,7 +312,20 @@ over the configuration files in the image.
 
 You can set individual {es} configuration parameters using Docker environment variables. 
 The <<docker-compose-file, sample compose file>> and the 
-<<docker-cli-run-dev-mode, single-node example>>  use this method. 
+<<docker-cli-run-dev-mode, single-node example>> use this method.
+
+To use the contents of a file to set an environment variable, suffix the environment
+variable name with `_FILE`. This is useful for passing secrets such as passwords to {es}
+without specifying them directly.
+
+For example, to set the {es} bootstrap password from a file, you can bind mount the
+file and set the `ELASTIC_PASSWORD_FILE` environment variable to the mount location.
+If you mount the password file to `/run/secrets/password.txt`, specify:
+
+[source,sh]
+--------------------------------------------
+-e ELASTIC_PASSWORD_FILE=/run/secrets/bootstrapPassword.txt
+--------------------------------------------
 
 You can also override the default command for the image to pass {es} configuration
 parameters as command line options. For example:

qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java

@@ -25,10 +25,12 @@ import org.elasticsearch.packaging.util.Docker.DockerShell;
 import org.elasticsearch.packaging.util.Installation;
 import org.elasticsearch.packaging.util.ServerUtils;
 import org.elasticsearch.packaging.util.Shell.Result;
+import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
 
+import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Map;
@@ -40,22 +42,25 @@ import static org.elasticsearch.packaging.util.Docker.ensureImageIsLoaded;
 import static org.elasticsearch.packaging.util.Docker.existsInContainer;
 import static org.elasticsearch.packaging.util.Docker.removeContainer;
 import static org.elasticsearch.packaging.util.Docker.runContainer;
+import static org.elasticsearch.packaging.util.Docker.runContainerExpectingFailure;
 import static org.elasticsearch.packaging.util.Docker.verifyContainerInstallation;
+import static org.elasticsearch.packaging.util.Docker.waitForElasticsearch;
 import static org.elasticsearch.packaging.util.Docker.waitForPathToExist;
+import static org.elasticsearch.packaging.util.FileMatcher.p600;
 import static org.elasticsearch.packaging.util.FileMatcher.p660;
 import static org.elasticsearch.packaging.util.FileUtils.append;
 import static org.elasticsearch.packaging.util.FileUtils.getTempDir;
-import static org.elasticsearch.packaging.util.FileUtils.mkdir;
 import static org.elasticsearch.packaging.util.FileUtils.rm;
 import static org.elasticsearch.packaging.util.ServerUtils.makeRequest;
-import static org.elasticsearch.packaging.util.ServerUtils.waitForElasticsearch;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.Matchers.emptyString;
+import static org.hamcrest.Matchers.equalTo;
 import static org.junit.Assume.assumeTrue;
 
 public class DockerTests extends PackagingTestCase {
     protected DockerShell sh;
+    private Path tempDir;
 
     @BeforeClass
     public static void filterDistros() {
@@ -71,9 +76,15 @@ public class DockerTests extends PackagingTestCase {
     }
 
     @Before
-    public void setupTest() throws Exception {
+    public void setupTest() throws IOException {
         sh = new DockerShell();
         installation = runContainer(distribution());
+        tempDir = Files.createTempDirectory(getTempDir(), DockerTests.class.getSimpleName());
+    }
+
+    @After
+    public void teardownTest() {
+        rm(tempDir);
     }
 
     /**
@@ -143,39 +154,156 @@ public class DockerTests extends PackagingTestCase {
      * Check that the default config can be overridden using a bind mount, and that env vars are respected
      */
     public void test70BindMountCustomPathConfAndJvmOptions() throws Exception {
-        final Path tempConf = getTempDir().resolve("esconf-alternate");
+        copyFromContainer(installation.config("elasticsearch.yml"), tempDir.resolve("elasticsearch.yml"));
+        copyFromContainer(installation.config("log4j2.properties"), tempDir.resolve("log4j2.properties"));
+
+        // we have to disable Log4j from using JMX lest it will hit a security
+        // manager exception before we have configured logging; this will fail
+        // startup since we detect usages of logging before it is configured
+        final String jvmOptions = "-Xms512m\n-Xmx512m\n-Dlog4j2.disable.jmx=true\n";
+        append(tempDir.resolve("jvm.options"), jvmOptions);
+
+        // Make the temp directory and contents accessible when bind-mounted
+        Files.setPosixFilePermissions(tempDir, fromString("rwxrwxrwx"));
+
+        // Restart the container
+        final Map<Path, Path> volumes = Map.of(tempDir, Path.of("/usr/share/elasticsearch/config"));
+        runContainer(distribution(), volumes, Map.of("ES_JAVA_OPTS", "-XX:-UseCompressedOops"));
+
+        waitForElasticsearch(installation);
+
+        final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes"));
+        assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":536870912"));
+        assertThat(nodesResponse, containsString("\"using_compressed_ordinary_object_pointers\":\"false\""));
+    }
+
+    /**
+     * Check that environment variables can be populated by setting variables with the suffix "_FILE",
+     * which point to files that hold the required values.
+     */
+    public void test80SetEnvironmentVariablesUsingFiles() throws Exception {
+        final String optionsFilename = "esJavaOpts.txt";
+
+        // ES_JAVA_OPTS_FILE
+        Files.writeString(tempDir.resolve(optionsFilename), "-XX:-UseCompressedOops\n");
+
+        Map<String, String> envVars = Map.of("ES_JAVA_OPTS_FILE", "/run/secrets/" + optionsFilename);
+
+        // File permissions need to be secured in order for the ES wrapper to accept
+        // them for populating env var values
+        Files.setPosixFilePermissions(tempDir.resolve(optionsFilename), p600);
+
+        final Map<Path, Path> volumes = Map.of(tempDir, Path.of("/run/secrets"));
+
+        // Restart the container
+        runContainer(distribution(), volumes, envVars);
+
+        waitForElasticsearch(installation);
+
+        final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes"));
+
+        assertThat(nodesResponse, containsString("\"using_compressed_ordinary_object_pointers\":\"false\""));
+    }
+
+    /**
+     * Check that the elastic user's password can be configured via a file and the ELASTIC_PASSWORD_FILE environment variable.
+     */
+    public void test81ConfigurePasswordThroughEnvironmentVariableFile() throws Exception {
+        // Test relies on configuring security
+        assumeTrue(distribution.isDefault());
+
+        final String xpackPassword = "hunter2";
+        final String passwordFilename = "password.txt";
 
+        // ELASTIC_PASSWORD_FILE
+        Files.writeString(tempDir.resolve(passwordFilename), xpackPassword + "\n");
+
+        Map<String, String> envVars = Map
+            .of(
+                "ELASTIC_PASSWORD_FILE",
+                "/run/secrets/" + passwordFilename,
+                // Enable security so that we can test that the password has been used
+                "xpack.security.enabled",
+                "true"
+            );
+
+        // File permissions need to be secured in order for the ES wrapper to accept
+        // them for populating env var values
+        Files.setPosixFilePermissions(tempDir.resolve(passwordFilename), p600);
+
+        final Map<Path, Path> volumes = Map.of(tempDir, Path.of("/run/secrets"));
+
+        // Restart the container
+        runContainer(distribution(), volumes, envVars);
+
+        // If we configured security correctly, then this call will only work if we specify the correct credentials.
         try {
-            mkdir(tempConf);
-            copyFromContainer(installation.config("elasticsearch.yml"), tempConf.resolve("elasticsearch.yml"));
-            copyFromContainer(installation.config("log4j2.properties"), tempConf.resolve("log4j2.properties"));
-
-            // we have to disable Log4j from using JMX lest it will hit a security
-            // manager exception before we have configured logging; this will fail
-            // startup since we detect usages of logging before it is configured
-            final String jvmOptions =
-                "-Xms512m\n" +
-                "-Xmx512m\n" +
-                "-Dlog4j2.disable.jmx=true\n";
-            append(tempConf.resolve("jvm.options"), jvmOptions);
-
-            // Make the temp directory and contents accessible when bind-mounted
-            Files.setPosixFilePermissions(tempConf, fromString("rwxrwxrwx"));
-
-            // Restart the container
-            removeContainer();
-            runContainer(distribution(), tempConf, Map.of(
-                "ES_JAVA_OPTS", "-XX:-UseCompressedOops"
-            ));
-
-            waitForElasticsearch(installation);
-
-            final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes"));
-            assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":536870912"));
-            assertThat(nodesResponse, containsString("\"using_compressed_ordinary_object_pointers\":\"false\""));
-        } finally {
-            rm(tempConf);
+            waitForElasticsearch("green", null, installation, "elastic", "hunter2");
+        } catch (Exception e) {
+            throw new AssertionError(
+                "Failed to check whether Elasticsearch had started. This could be because "
+                    + "authentication isn't working properly. Check the container logs",
+                e
+            );
         }
+
+        // Also check that an unauthenticated call fails
+        final int statusCode = Request.Get("http://localhost:9200/_nodes").execute().returnResponse().getStatusLine().getStatusCode();
+        assertThat("Expected server to require authentication", statusCode, equalTo(401));
+    }
+
+    /**
+     * Check that environment variables cannot be used with _FILE environment variables.
+     */
+    public void test81CannotUseEnvVarsAndFiles() throws Exception {
+        final String optionsFilename = "esJavaOpts.txt";
+
+        // ES_JAVA_OPTS_FILE
+        Files.writeString(tempDir.resolve(optionsFilename), "-XX:-UseCompressedOops\n");
+
+        Map<String, String> envVars = Map.of(
+            "ES_JAVA_OPTS", "-XX:+UseCompressedOops",
+            "ES_JAVA_OPTS_FILE", "/run/secrets/" + optionsFilename
+        );
+
+        // File permissions need to be secured in order for the ES wrapper to accept
+        // them for populating env var values
+        Files.setPosixFilePermissions(tempDir.resolve(optionsFilename), p600);
+
+        final Map<Path, Path> volumes = Map.of(tempDir, Path.of("/run/secrets"));
+
+        final Result dockerLogs = runContainerExpectingFailure(distribution, volumes, envVars);
+
+        assertThat(
+            dockerLogs.stderr,
+            containsString("ERROR: Both ES_JAVA_OPTS_FILE and ES_JAVA_OPTS are set. These are mutually exclusive.")
+        );
+    }
+
+    /**
+     * Check that when populating environment variables by setting variables with the suffix "_FILE",
+     * the files' permissions are checked.
+     */
+    public void test82EnvironmentVariablesUsingFilesHaveCorrectPermissions() throws Exception {
+        final String optionsFilename = "esJavaOpts.txt";
+
+        // ES_JAVA_OPTS_FILE
+        Files.writeString(tempDir.resolve(optionsFilename), "-XX:-UseCompressedOops\n");
+
+        Map<String, String> envVars = Map.of("ES_JAVA_OPTS_FILE", "/run/secrets/" + optionsFilename);
+
+        // Set invalid file permissions
+        Files.setPosixFilePermissions(tempDir.resolve(optionsFilename), p660);
+
+        final Map<Path, Path> volumes = Map.of(tempDir, Path.of("/run/secrets"));
+
+        // Restart the container
+        final Result dockerLogs = runContainerExpectingFailure(distribution(), volumes, envVars);
+
+        assertThat(
+            dockerLogs.stderr,
+            containsString("ERROR: File /run/secrets/" + optionsFilename + " from ES_JAVA_OPTS_FILE must have file permissions 400 or 600")
+        );
     }
 
     /**
@@ -219,7 +347,6 @@ public class DockerTests extends PackagingTestCase {
         final Installation.Executables bin = installation.executables();
 
         final Result result = sh.run(bin.elasticsearchNode + " -h");
-        assertThat(result.stdout,
-            containsString("A CLI tool to do unsafe cluster and index manipulations on current node"));
+        assertThat(result.stdout, containsString("A CLI tool to do unsafe cluster and index manipulations on current node"));
     }
 }

qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java

@@ -25,7 +25,6 @@ import org.apache.commons.logging.LogFactory;
 import java.nio.file.Path;
 import java.nio.file.attribute.PosixFilePermission;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -79,8 +78,8 @@ public class Docker {
      * Runs an Elasticsearch Docker container.
      * @param distribution details about the docker image being tested.
      */
-    public static Installation runContainer(Distribution distribution) throws Exception {
-        return runContainer(distribution, null, Collections.emptyMap());
+    public static Installation runContainer(Distribution distribution) {
+        return runContainer(distribution, null, null);
     }
 
     /**
@@ -88,23 +87,51 @@ public class Docker {
      * through a bind mount, and passing additional environment variables.
      *
      * @param distribution details about the docker image being tested.
-     * @param configPath the path to the config to bind mount, or null
-     * @param envVars environment variables to set when running the container
+     * @param volumes a map that declares any volume mappings to apply, or null
+     * @param envVars environment variables to set when running the container, or null
      */
-    public static Installation runContainer(Distribution distribution, Path configPath, Map<String,String> envVars) throws Exception {
+    public static Installation runContainer(Distribution distribution, Map<Path, Path> volumes, Map<String, String> envVars) {
+        executeDockerRun(distribution, volumes, envVars);
+
+        waitForElasticsearchToStart();
+
+        return Installation.ofContainer();
+    }
+
+    /**
+     * Similar to {@link #runContainer(Distribution, Map, Map)} in that it runs an Elasticsearch Docker
+     * container, expect that the container expecting it to exit e.g. due to configuration problem.
+     *
+     * @param distribution details about the docker image being tested.
+     * @param volumes a map that declares any volume mappings to apply, or null
+     * @param envVars environment variables to set when running the container, or null
+     * @return the docker logs of the container
+     */
+    public static Shell.Result runContainerExpectingFailure(
+        Distribution distribution,
+        Map<Path, Path> volumes,
+        Map<String, String> envVars
+    ) {
+        executeDockerRun(distribution, volumes, envVars);
+
+        waitForElasticsearchToExit();
+
+        return sh.run("docker logs " + containerId);
+    }
+
+    private static void executeDockerRun(Distribution distribution, Map<Path, Path> volumes, Map<String, String> envVars) {
         removeContainer();
 
         final List<String> args = new ArrayList<>();
 
         args.add("docker run");
 
-        // Remove the container once it exits
-        args.add("--rm");
-
         // Run the container in the background
         args.add("--detach");
 
-        envVars.forEach((key, value) -> args.add("--env " + key + "=\"" + value + "\""));
+        if (envVars != null) {
+            envVars.forEach((key, value) -> args.add("--env " + key + "=\"" + value + "\""));
+        }
 
         // The container won't run without configuring discovery
         args.add("--env discovery.type=single-node");
@@ -113,48 +140,81 @@ public class Docker {
         args.add("--publish 9200:9200");
         args.add("--publish 9300:9300");
 
-        if (configPath != null) {
-            // Bind-mount the config dir, if specified
-            args.add("--volume \"" + configPath + ":/usr/share/elasticsearch/config\"");
+        // Bind-mount any volumes
+        if (volumes != null) {
+            volumes.forEach((localPath, containerPath) -> args.add("--volume \"" + localPath + ":" + containerPath + "\""));
         }
 
         args.add(distribution.flavor.name + ":test");
 
         final String command = String.join(" ", args);
-        logger.debug("Running command: " + command);
+        logger.info("Running command: " + command);
         containerId = sh.run(command).stdout.trim();
-
-        waitForElasticsearchToStart();
-
-        return Installation.ofContainer();
     }
 
     /**
      * Waits for the Elasticsearch process to start executing in the container.
      * This is called every time a container is started.
      */
-    private static void waitForElasticsearchToStart() throws InterruptedException {
+    private static void waitForElasticsearchToStart() {
         boolean isElasticsearchRunning = false;
         int attempt = 0;
 
-        String psOutput;
+        String psOutput = null;
 
         do {
-            // Give the container a chance to crash out
-            Thread.sleep(1000);
+            try {
+                // Give the container a chance to crash out
+                Thread.sleep(1000);
 
-            psOutput = dockerShell.run("ps ax").stdout;
+                psOutput = dockerShell.run("ps ax").stdout;
 
-            if (psOutput.contains("/usr/share/elasticsearch/jdk/bin/java")) {
-                isElasticsearchRunning = true;
-                break;
+                if (psOutput.contains("/usr/share/elasticsearch/jdk/bin/java")) {
+                    isElasticsearchRunning = true;
+                    break;
+                }
+            } catch (Exception e) {
+                logger.warn("Caught exception while waiting for ES to start", e);
             }
+        } while (attempt++ < 5);
+
+        if (isElasticsearchRunning == false) {
+            final Shell.Result dockerLogs = sh.run("docker logs " + containerId);
+            fail(
+                "Elasticsearch container did not start successfully.\n\nps output:\n"
+                    + psOutput
+                    + "\n\nStdout:\n"
+                    + dockerLogs.stdout
+                    + "\n\nStderr:\n"
+                    + dockerLogs.stderr
+            );
+        }
+    }
+
+    /**
+     * Waits for the Elasticsearch container to exit.
+     */
+    private static void waitForElasticsearchToExit() {
+        boolean isElasticsearchRunning = true;
+        int attempt = 0;
+
+        do {
+            try {
+                // Give the container a chance to exit out
+                Thread.sleep(1000);
 
+                if (sh.run("docker ps --quiet --no-trunc").stdout.contains(containerId) == false) {
+                    isElasticsearchRunning = false;
+                    break;
+                }
+            } catch (Exception e) {
+                logger.warn("Caught exception while waiting for ES to exit", e);
+            }
         } while (attempt++ < 5);
 
-        if (!isElasticsearchRunning) {
-            final String dockerLogs = sh.run("docker logs " + containerId).stdout;
-            fail("Elasticsearch container did start successfully.\n\n" + psOutput + "\n\n" + dockerLogs);
+        if (isElasticsearchRunning) {
+            final Shell.Result dockerLogs = sh.run("docker logs " + containerId);
+            fail("Elasticsearch container did exit.\n\nStdout:\n" + dockerLogs.stdout + "\n\nStderr:\n" + dockerLogs.stderr);
         }
     }
 
@@ -170,10 +230,12 @@ public class Docker {
                 final Shell.Result result = sh.runIgnoreExitCode(command);
 
                 if (result.isSuccess() == false) {
+                    boolean isErrorAcceptable = result.stderr.contains("removal of container " + containerId + " is already in progress")
+                        || result.stderr.contains("Error: No such container: " + containerId);
+
                     // I'm not sure why we're already removing this container, but that's OK.
-                    if (result.stderr.contains("removal of container " + " is already in progress") == false) {
-                        throw new RuntimeException(
-                            "Command was not successful: [" + command + "] result: " + result.toString());
+                    if (isErrorAcceptable == false) {
+                        throw new RuntimeException("Command was not successful: [" + command + "] result: " + result.toString());
                     }
                 }
             } finally {
@@ -204,11 +266,7 @@ public class Docker {
         protected String[] getScriptCommand(String script) {
             assert containerId != null;
 
-            return super.getScriptCommand("docker exec " +
-                "--user elasticsearch:root " +
-                "--tty " +
-                containerId + " " +
-                script);
+            return super.getScriptCommand("docker exec " + "--user elasticsearch:root " + "--tty " + containerId + " " + script);
         }
     }
 
@@ -278,82 +336,90 @@ public class Docker {
         final String homeDir = passwdResult.stdout.trim().split(":")[5];
         assertThat(homeDir, equalTo("/usr/share/elasticsearch"));
 
-        Stream.of(
-            es.home,
-            es.data,
-            es.logs,
-            es.config
-        ).forEach(dir -> assertPermissionsAndOwnership(dir, p775));
+        Stream.of(es.home, es.data, es.logs, es.config).forEach(dir -> assertPermissionsAndOwnership(dir, p775));
 
-        Stream.of(
-            es.plugins,
-            es.modules
-        ).forEach(dir -> assertPermissionsAndOwnership(dir, p755));
+        Stream.of(es.plugins, es.modules).forEach(dir -> assertPermissionsAndOwnership(dir, p755));
 
         // FIXME these files should all have the same permissions
-        Stream.of(
-            "elasticsearch.keystore",
-//            "elasticsearch.yml",
-            "jvm.options"
-//            "log4j2.properties"
-        ).forEach(configFile -> assertPermissionsAndOwnership(es.config(configFile), p660));
-
-        Stream.of(
-            "elasticsearch.yml",
-            "log4j2.properties"
-        ).forEach(configFile -> assertPermissionsAndOwnership(es.config(configFile), p644));
-
-        assertThat(
-            dockerShell.run(es.bin("elasticsearch-keystore") + " list").stdout,
-            containsString("keystore.seed"));
-
-        Stream.of(
-            es.bin,
-            es.lib
-        ).forEach(dir -> assertPermissionsAndOwnership(dir, p755));
-
-        Stream.of(
-            "elasticsearch",
-            "elasticsearch-cli",
-            "elasticsearch-env",
-            "elasticsearch-enve",
-            "elasticsearch-keystore",
-            "elasticsearch-node",
-            "elasticsearch-plugin",
-            "elasticsearch-shard"
-        ).forEach(executable -> assertPermissionsAndOwnership(es.bin(executable), p755));
-
-        Stream.of(
-            "LICENSE.txt",
-            "NOTICE.txt",
-            "README.textile"
-        ).forEach(doc -> assertPermissionsAndOwnership(es.home.resolve(doc), p644));
+        Stream
+            .of(
+                "elasticsearch.keystore",
+                // "elasticsearch.yml",
+                "jvm.options"
+                // "log4j2.properties"
+            )
+            .forEach(configFile -> assertPermissionsAndOwnership(es.config(configFile), p660));
+
+        Stream
+            .of("elasticsearch.yml", "log4j2.properties")
+            .forEach(configFile -> assertPermissionsAndOwnership(es.config(configFile), p644));
+
+        assertThat(dockerShell.run(es.bin("elasticsearch-keystore") + " list").stdout, containsString("keystore.seed"));
+
+        Stream.of(es.bin, es.lib).forEach(dir -> assertPermissionsAndOwnership(dir, p755));
+
+        Stream
+            .of(
+                "elasticsearch",
+                "elasticsearch-cli",
+                "elasticsearch-env",
+                "elasticsearch-enve",
+                "elasticsearch-keystore",
+                "elasticsearch-node",
+                "elasticsearch-plugin",
+                "elasticsearch-shard"
+            )
+            .forEach(executable -> assertPermissionsAndOwnership(es.bin(executable), p755));
+
+        Stream.of("LICENSE.txt", "NOTICE.txt", "README.textile").forEach(doc -> assertPermissionsAndOwnership(es.home.resolve(doc), p644));
     }
 
     private static void verifyDefaultInstallation(Installation es) {
-        Stream.of(
-            "elasticsearch-certgen",
-            "elasticsearch-certutil",
-            "elasticsearch-croneval",
-            "elasticsearch-saml-metadata",
-            "elasticsearch-setup-passwords",
-            "elasticsearch-sql-cli",
-            "elasticsearch-syskeygen",
-            "elasticsearch-users",
-            "x-pack-env",
-            "x-pack-security-env",
-            "x-pack-watcher-env"
-        ).forEach(executable -> assertPermissionsAndOwnership(es.bin(executable), p755));
+        Stream
+            .of(
+                "elasticsearch-certgen",
+                "elasticsearch-certutil",
+                "elasticsearch-croneval",
+                "elasticsearch-saml-metadata",
+                "elasticsearch-setup-passwords",
+                "elasticsearch-sql-cli",
+                "elasticsearch-syskeygen",
+                "elasticsearch-users",
+                "x-pack-env",
+                "x-pack-security-env",
+                "x-pack-watcher-env"
+            )
+            .forEach(executable -> assertPermissionsAndOwnership(es.bin(executable), p755));
 
         // at this time we only install the current version of archive distributions, but if that changes we'll need to pass
         // the version through here
         assertPermissionsAndOwnership(es.bin("elasticsearch-sql-cli-" + getCurrentVersion() + ".jar"), p755);
 
-        Stream.of(
-            "role_mapping.yml",
-            "roles.yml",
-            "users",
-            "users_roles"
-        ).forEach(configFile -> assertPermissionsAndOwnership(es.config(configFile), p660));
+        Stream
+            .of("role_mapping.yml", "roles.yml", "users", "users_roles")
+            .forEach(configFile -> assertPermissionsAndOwnership(es.config(configFile), p660));
+    }
+
+    public static void waitForElasticsearch(Installation installation) throws Exception {
+        withLogging(() -> ServerUtils.waitForElasticsearch(installation));
+    }
+
+    public static void waitForElasticsearch(String status, String index, Installation installation, String username, String password)
+        throws Exception {
+        withLogging(() -> ServerUtils.waitForElasticsearch(status, index, installation, username, password));
+    }
+
+    private static void withLogging(ThrowingRunnable r) throws Exception {
+        try {
+            r.run();
+        } catch (Exception e) {
+            final Shell.Result logs = sh.run("docker logs " + containerId);
+            logger.warn("Elasticsearch container failed to start.\n\nStdout:\n" + logs.stdout + "\n\nStderr:\n" + logs.stderr);
+            throw e;
+        }
+    }
+
+    private interface ThrowingRunnable {
+        void run() throws Exception;
     }
 }

qa/os/src/test/java/org/elasticsearch/packaging/util/FileMatcher.java

@@ -50,6 +50,7 @@ public class FileMatcher extends TypeSafeMatcher<Path> {
     public static final Set<PosixFilePermission> p750 = fromString("rwxr-x---");
     public static final Set<PosixFilePermission> p660 = fromString("rw-rw----");
     public static final Set<PosixFilePermission> p644 = fromString("rw-r--r--");
+    public static final Set<PosixFilePermission> p600 = fromString("rw-------");
 
     private final Fileness fileness;
     private final String owner;

qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java

@@ -19,7 +19,9 @@
 
 package org.elasticsearch.packaging.util;
 
+import org.apache.http.HttpHost;
 import org.apache.http.HttpResponse;
+import org.apache.http.client.fluent.Executor;
 import org.apache.http.client.fluent.Request;
 import org.apache.http.entity.ContentType;
 import org.apache.http.util.EntityUtils;
@@ -35,7 +37,7 @@ import static org.hamcrest.Matchers.containsString;
 
 public class ServerUtils {
 
-    protected static final Logger logger =  LogManager.getLogger(ServerUtils.class);
+    private static final Logger logger =  LogManager.getLogger(ServerUtils.class);
 
     // generous timeout  as nested virtualization can be quite slow ...
     private static final long waitTime = TimeUnit.MINUTES.toMillis(3);
@@ -43,10 +45,36 @@ public class ServerUtils {
     private static final long requestInterval = TimeUnit.SECONDS.toMillis(5);
 
     public static void waitForElasticsearch(Installation installation) throws IOException {
-        waitForElasticsearch("green", null, installation);
+        waitForElasticsearch("green", null, installation, null, null);
     }
 
-    public static void waitForElasticsearch(String status, String index, Installation installation) throws IOException {
+    /**
+     * Executes the supplied request, optionally applying HTTP basic auth if the
+     * username and pasword field are supplied.
+     * @param request the request to execute
+     * @param username the username to supply, or null
+     * @param password the password to supply, or null
+     * @return the response from the server
+     * @throws IOException if an error occurs
+     */
+    private static HttpResponse execute(Request request, String username, String password) throws IOException {
+        final Executor executor = Executor.newInstance();
+
+        if (username != null && password != null) {
+            executor.auth(username, password);
+            executor.authPreemptive(new HttpHost("localhost", 9200));
+        }
+
+        return executor.execute(request).returnResponse();
+    }
+
+    public static void waitForElasticsearch(
+        String status,
+        String index,
+        Installation installation,
+        String username,
+        String password
+    ) throws IOException {
 
         Objects.requireNonNull(status);
 
@@ -56,15 +84,19 @@ public class ServerUtils {
         long timeElapsed = 0;
         boolean started = false;
         Throwable thrownException = null;
+
         while (started == false && timeElapsed < waitTime) {
             if (System.currentTimeMillis() - lastRequest > requestInterval) {
                 try {
 
-                    final HttpResponse response = Request.Get("http://localhost:9200/_cluster/health")
-                        .connectTimeout((int) timeoutLength)
-                        .socketTimeout((int) timeoutLength)
-                        .execute()
-                        .returnResponse();
+                    final HttpResponse response = execute(
+                        Request
+                            .Get("http://localhost:9200/_cluster/health")
+                            .connectTimeout((int) timeoutLength)
+                            .socketTimeout((int) timeoutLength),
+                        username,
+                        password
+                    );
 
                     if (response.getStatusLine().getStatusCode() >= 300) {
                         final String statusLine = response.getStatusLine().toString();
@@ -101,10 +133,9 @@ public class ServerUtils {
             url = "http://localhost:9200/_cluster/health?wait_for_status=" + status + "&timeout=60s&pretty";
         } else {
             url = "http://localhost:9200/_cluster/health/" + index + "?wait_for_status=" + status + "&timeout=60s&pretty";
-
         }
 
-        final String body = makeRequest(Request.Get(url));
+        final String body = makeRequest(Request.Get(url), username, password);
         assertThat("cluster health response must contain desired status", body, containsString(status));
     }
 
@@ -124,7 +155,11 @@ public class ServerUtils {
     }
 
     public static String makeRequest(Request request) throws IOException {
-        final HttpResponse response = request.execute().returnResponse();
+        return makeRequest(request, null, null);
+    }
+
+    public static String makeRequest(Request request, String username, String password) throws IOException {
+        final HttpResponse response = execute(request, username, password);
         final String body = EntityUtils.toString(response.getEntity());
 
         if (response.getStatusLine().getStatusCode() >= 300) {
@@ -132,6 +167,5 @@ public class ServerUtils {
         }
 
         return body;
-
     }
 }